def lock(facets_group,model):
verbose=sddquery.get_scalar(facets_group,'verbose',default=False,type_=bool) # we cast here as verbose can be str (set from parameter) or bool (set from '-v' option)
if model in cache:
facets_group["searchapi_host"]=cache[model]
facets_group["distrib"]=['false']
else:
params=sdremoteparam.run(pname='index_node',facets_group={'type':['Dataset'],'model':[model],'replica':['false']},dry_run=False)
indexes=params.get('index_node',[])
if verbose:
sdtools.print_stderr('Master index candidates for %s model are:'%model)
for i in indexes:
sdtools.print_stderr('%s %i'%(i.name,i.count))
if len(indexes)>0:
master_index=max(indexes, key=attrgetter('count'))
if verbose:
sdtools.print_stderr("'searchapi_host' has been set to %s"%master_index)
facets_group["searchapi_host"]=master_index.name
facets_group["distrib"]=['false']
cache[model]=master_index.name
def facet(args):
import sdparam,sdremoteparam,syndautils,sdinference,sdignorecase
facets_groups=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file,no_default=True)
facets_groups=sdignorecase.run(facets_groups)
facets_groups=sdinference.run(facets_groups)
if sdparam.exists_parameter_name(args.facet_name): # first, we check in cache so to quickly return if facet is unknown
if len(facets_groups)==1:
# facet selected: retrieve parameters from ESGF
facets_group=facets_groups[0]
params=sdremoteparam.run(pname=args.facet_name,facets_group=facets_group,dry_run=args.dry_run)
# TODO: func for code below
items=params.get(args.facet_name,[])
for item in items:
print item.name
elif len(facets_groups)>1:
print_stderr('Multi-queries not supported')
else:
# Parameter not set. In this case, we retrieve facet values list from cache.
sdparam.main([args.facet_name]) # tricks to re-use sdparam CLI parser
else:
print_stderr('Unknown facet')
def dataset_version(args):
import sdremoteparam, syndautils, sdearlystreamutils
# don't be misled about identifiers here: sdinference produces search-api
# key (always do) name i.e. instance_id, and I use Synda style variable name
# i.e. dataset_functional_id (for better readability).
li = sdearlystreamutils.get_facet_values_early(args.stream, 'instance_id')
if len(li) == 0:
print_stderr('Please specify a dataset name.')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
else:
dataset_functional_id = li[0]
dataset_functional_id_without_version = syndautils.strip_dataset_version(
dataset_functional_id)
params = sdremoteparam.run(pname='version',
facets_group={
'type': [sdconst.SA_TYPE_DATASET],
'master_id':
[dataset_functional_id_without_version]
},
dry_run=args.dry_run)
# TODO: func for code below
items = params.get('version', [])
for item in items:
print item.name
def lock(facets_group,model):
verbose=sddquery.get_scalar(facets_group,'verbose',default=False,type_=bool) # we cast here as verbose can be str (set from parameter) or bool (set from '-v' option)
if model in cache:
facets_group["searchapi_host"]=cache[model]
facets_group["distrib"]=['false']
else:
params=sdremoteparam.run(pname='index_node',facets_group={'type':['Dataset'],'model':[model],'replica':['false']},dry_run=False)
indexes=params.get('index_node',[])
if verbose:
sdtools.print_stderr('Master index candidates for %s model are:'%model)
for i in indexes:
sdtools.print_stderr('%s %i'%(i.name,i.count))
if len(indexes)>0:
master_index=max(indexes, key=attrgetter('count'))
if verbose:
sdtools.print_stderr("'searchapi_host' has been set to %s"%master_index)
facets_group["searchapi_host"]=master_index.name
facets_group["distrib"]=['false']
cache[model]=master_index.name
def default(self, arg):
"""
This func add support to automatically list available parameters, depending on
which facets are currently selected.
"""
if sdparam.exists_parameter_name(arg):
selected_facets = sdsessionparam.get_session_facets_as_facetsgroup(
)
if len(selected_facets) > 0:
# facet selected: retrieve parameters from ESGF
params = sdremoteparam.run(pname=arg,
facets_group=selected_facets,
dry_run=True)
# TODO: func for code below
items = params.get(arg, [])
for item in items:
print item.name
else:
# no facet selected: retrieve parameters from cache
sdparam.main(
arg.split()) # tricks to re-use sdparam CLI parser
else:
print '*** Unknown command: %s' % arg
def lock(model):
dry_run=sdsessionparam.get_value('dry_run')
params=sdremoteparam.run(pname='index_node',facets_group={'type':['Dataset'],'model':[model],'replica':['false']},dry_run=dry_run)
indexes=params.get('index_node',[])
if not dry_run:
if len(indexes)>0:
master_index=max(indexes, key=attrgetter('count'))
sdsessionparam.set("searchapi_host",master_index.name)
sdsessionparam.set("distrib","false")
sdsessionparam.set("model",model)
else:
raise SDException('SDATLOCK-001','Model not found')
def lock(model):
dry_run = sdsessionparam.get_value('dry_run')
params = sdremoteparam.run(pname='index_node',
facets_group={
'type': ['Dataset'],
'model': [model],
'replica': ['false']
},
dry_run=dry_run)
indexes = params.get('index_node', [])
if not dry_run:
if len(indexes) > 0:
master_index = max(indexes, key=attrgetter('count'))
sdsessionparam.set("searchapi_host", master_index.name)
sdsessionparam.set("distrib", "false")
sdsessionparam.set("model", model)
else:
raise SDException('SDATLOCK-001', 'Model not found')
def facet(args):
import sdparam, sdremoteparam, syndautils, sdinference, sdignorecase
facets_groups = syndautils.get_stream(subcommand=args.subcommand,
parameter=args.parameter,
selection_file=args.selection_file,
no_default=True)
facets_groups = sdignorecase.run(facets_groups)
facets_groups = sdinference.run(facets_groups)
if sdparam.exists_parameter_name(
args.facet_name
): # first, we check in cache so to quickly return if facet is unknown
if len(facets_groups) == 1:
# facet selected: retrieve parameters from ESGF
facets_group = facets_groups[0]
params = sdremoteparam.run(pname=args.facet_name,
facets_group=facets_group,
dry_run=args.dry_run)
# TODO: func for code below
items = params.get(args.facet_name, [])
for item in items:
print item.name
elif len(facets_groups) > 1:
print_stderr('Multi-queries not supported')
else:
# Parameter not set. In this case, we retrieve facet values list from cache.
sdparam.main([args.facet_name
]) # tricks to re-use sdparam CLI parser
else:
print_stderr('Unknown facet')
def default(self,arg):
"""
This func add support to automatically list available parameters, depending on
which facets are currently selected.
"""
if sdparam.exists_parameter_name(arg):
selected_facets=sdsessionparam.get_session_facets_as_facetsgroup()
if len(selected_facets)>0:
# facet selected: retrieve parameters from ESGF
params=sdremoteparam.run(pname=arg,facets_group=selected_facets,dry_run=True)
# TODO: func for code below
items=params.get(arg,[])
for item in items:
print item.name
else:
# no facet selected: retrieve parameters from cache
sdparam.main(arg.split()) # tricks to re-use sdparam CLI parser
else:
print '*** Unknown command: %s'%arg
def dataset_version(args):
import sdremoteparam
# don't be misled about identifiers here: sdinference produces search-api
# key (always do) name i.e. instance_id, and I use Synda style variable name
# i.e. dataset_functional_id (for better readability).
li=syndautils.get_facet_values_early(args.stream,'instance_id')
if len(li)==0:
print_stderr('Please specify a dataset name.')
return
elif len(li)>1:
print_stderr('Too many arguments.')
return
else:
dataset_functional_id=li[0]
dataset_functional_id_without_version=syndautils.strip_dataset_version(dataset_functional_id)
params=sdremoteparam.run(pname='version',facets_group={'type':[sdconst.SA_TYPE_DATASET],'master_id':[dataset_functional_id_without_version]},dry_run=args.dry_run)
# TODO: func for code below
items=params.get('version',[])
for item in items:
print item.name
def get_datanode_list(facets_group, dry_run=False):
params = sdremoteparam.run(pname='data_node',
facets_group=facets_group,
dry_run=dry_run)
items = params.get('data_node', [])
return [i.name for i in items]
def variable(args):
import sdremoteparam,sdutils,sdproxy_ra
# currently, mode (list or show) is determined by
# parameter existency. This may change in the future
# as it may be useful to list variable based on filter
# (e.g. list variable long name only for obs4MIPs
# project, etc..). To do that, we will need to
# add an 'action' argument (i.e. list and show).
#
action='show' if len(args.parameter)>0 else 'list'
if action=='list':
if args.long_name:
facet='variable_long_name'
elif args.short_name:
facet='variable'
elif args.standard_name:
facet='cf_standard_name'
else:
# no options set by user
facet='variable_long_name' # default
params=sdremoteparam.run(pname=facet,dry_run=args.dry_run)
if not args.dry_run:
# This try/except block is to prevent
# IOError: [Errno 32] Broken pipe
# Other way to prevent it is to ignore SIGPIPE
# More info at
# http://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
try:
# TODO: func for code below
items=params.get(facet)
for item in items:
print item.name
except:
pass
elif action=='show':
# We do not use inference here, instead we use
# search-api 'query' feature to do the job.
#
query=sdutils.parameter_to_query(args.parameter)
file_=sdproxy_ra.get_one_file(query=query,dry_run=args.dry_run)
if not args.dry_run:
if file_ is None:
print 'Variable not found.'
else:
print 'short name: ',file_['variable'][0]
print 'standard name: ',file_['cf_standard_name'][0]
print 'long name: ',file_['variable_long_name'][0]
print 'unit: ',file_['variable_units'][0]
def variable(args):
import sdremoteparam, sdutils, sdproxy_ra
# currently, mode (list or show) is determined by
# parameter existency. This may change in the future
# as it may be useful to list variable based on filter
# (e.g. list variable long name only for obs4MIPs
# project, etc..). To do that, we will need to
# add an 'action' argument (i.e. list and show).
#
action = 'show' if len(args.parameter) > 0 else 'list'
if action == 'list':
if args.long_name:
facet = 'variable_long_name'
elif args.short_name:
facet = 'variable'
elif args.standard_name:
facet = 'cf_standard_name'
else:
# no options set by user
facet = 'variable_long_name' # default
params = sdremoteparam.run(pname=facet, dry_run=args.dry_run)
if not args.dry_run:
# This try/except block is to prevent
# IOError: [Errno 32] Broken pipe
# Other way to prevent it is to ignore SIGPIPE
# More info at
# http://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
try:
# TODO: func for code below
items = params.get(facet)
for item in items:
print item.name
except:
pass
elif action == 'show':
# We do not use inference here, instead we use
# search-api 'query' feature to do the job.
#
query = sdutils.parameter_to_query(args.parameter)
file_ = sdproxy_ra.get_one_file(query=query, dry_run=args.dry_run)
if not args.dry_run:
if file_ is None:
print 'Variable not found.'
else:
print 'short name: ', file_['variable'][0]
print 'standard name: ', file_['cf_standard_name'][0]
print 'long name: ', file_['variable_long_name'][0]
print 'unit: ', file_['variable_units'][0]
def get_datanode_list(facets_group,dry_run=False):
params=sdremoteparam.run(pname='data_node',facets_group=facets_group,dry_run=dry_run)
items=params.get('data_node',[])
return [i.name for i in items]