def build_queries(stream=None,selection=None,path=None,parameter=None,index_host=None,load_default=None,query_type='remote',dry_run=False,parallel=True,count=False):
"""This pipeline add 'path', 'parameter' and 'selection' input type to the
standalone query pipeline.
Returns:
squeries (Serialized queries) # TODO: maybe rename stream to dqueries
"""
if parameter is None:
parameter=[]
if stream is None:
if selection is None:
buffer=sdbuffer.get_selection_file_buffer(path=path,parameter=parameter)
selection=sdparse.build(buffer,load_default=load_default)
stream=selection.merge_facets()
# at this point, stream contains all possible parameters sources (file,stdin,cli..)
if count:
# in this mode, we don't want to return any files, so we force limit to
# 0 just in case this option has been set by the user
sddeferredafter.add_forced_parameter(stream,'limit','0')
queries=sdquerypipeline.run(stream,index_host=index_host,query_type=query_type,dry_run=dry_run,parallel=parallel)
return queries
def get_stream(args,raise_exception_if_empty=False):
import sdbuffer, sdparse, sdstream, sdconfig, sddeferredbefore, sdexception
# hack
if args.subcommand=='list':
args.no_default=True
buffer=sdbuffer.get_selection_file_buffer(parameter=args.parameter,path=args.selection_file)
if len(buffer.lines)==0:
# we come here when user selected nothing (neither from stdin neither from selection file neither from arg)
if raise_exception_if_empty:
raise sdexception.EmptySelectionException()
selection=sdparse.build(buffer,load_default=(not args.no_default))
stream=selection.to_stream()
# Set default value for nearest here
#
# TODO: make it work with all actions (e.g. search) as it only working for 'install' action for now
#
#sddeferredbefore.add_default_parameter(stream,'nearest',True) # TODO: why this one is not working ?
if sdconfig.config.getboolean('behaviour','nearest'):
sdstream.set_scalar(stream,'nearest',True)
# progress
if sdconfig.config.getboolean('interface','progress'):
sdstream.set_scalar(stream,'progress',True)
sdstream.set_scalar(stream,'action',args.subcommand) # from the synda engine perspective, 'action' is more meaningful than 'subcommand'
return stream # aka facets_groups
def parse(parameter=[]):
"""This pipeline is used as a fast parameter parser (without further
processing).
"""
buffer=sdbuffer.get_selection_file_buffer(parameter=parameter)
selection=sdparse.build(buffer,load_default=False)
facets_groups=selection.merge_facets()
facets_groups=sdinference.run(facets_groups)
return facets_groups
def prepare_param(selection=None,path=None,parameter=[],load_default=None):
"""This pipeline add 'path', 'parameter' and 'selection' input type to the
standalone param pipeline."""
if selection is None:
buffer=sdbuffer.get_selection_file_buffer(path=path,parameter=parameter)
selection=sdparse.build(buffer,load_default=load_default)
facets_groups=selection.merge_facets()
facets_groups=sdparampipeline.run(facets_groups)
return facets_groups
def parse(parameter=None):
"""This pipeline is used as a fast parameter parser (without further
processing).
"""
if parameter is None:
parameter = []
buffer = sdbuffer.get_selection_file_buffer(parameter=parameter)
selection = sdparse.build(buffer, load_default=False)
facets_groups = selection.merge_facets()
facets_groups = sdignorecase.run(facets_groups)
facets_groups = sdinference.run(facets_groups)
return facets_groups
def build_selection_list():
"""
Return:
selections list.
"""
selections=[]
files=build_selection_file_list() # contains selection files path list (fullpath)
for file in files:
try:
buffer=sdbuffer.get_selection_file_buffer(path=file)
selection=sdparse.build(buffer)
selections.append(selection)
except Exception, e:
sdlog.error("SDSELGPU-001","Exception occured (%s)"%str(e))
raise SDException("SDSELGPU-001","Error occured while loading '%s' selection file. See log for details."%file)
def build_queries(stream=None,selection=None,path=None,parameter=[],index_host=None,load_default=None,query_type='remote',dry_run=False,parallel=True):
"""This pipeline add 'path', 'parameter' and 'selection' input type to the
standalone query pipeline.
Returns:
squeries (Serialized queries) # TODO: maybe rename stream to dqueries
"""
if stream is None:
if selection is None:
buffer=sdbuffer.get_selection_file_buffer(path=path,parameter=parameter)
selection=sdparse.build(buffer,load_default=load_default)
stream=selection.merge_facets()
queries=sdquerypipeline.run(stream,index_host=index_host,query_type=query_type,dry_run=dry_run,parallel=parallel)
return queries
def build_queries(stream=None,
selection=None,
path=None,
parameter=None,
index_host=None,
load_default=None,
query_type='remote',
dry_run=False,
parallel=True,
count=False):
"""This pipeline add 'path', 'parameter' and 'selection' input type to the
standalone query pipeline.
Returns:
squeries (Serialized queries) # TODO: maybe rename stream to dqueries
"""
if parameter is None:
parameter = []
if stream is None:
if selection is None:
buffer = sdbuffer.get_selection_file_buffer(path=path,
parameter=parameter)
selection = sdparse.build(buffer, load_default=load_default)
stream = selection.merge_facets()
# at this point, stream contains all possible parameters sources (file,stdin,cli..)
if count:
# in this mode, we don't want to return any files, so we force limit to
# 0 just in case this option has been set by the user
sddeferredafter.add_forced_parameter(stream, 'limit', '0')
queries = sdquerypipeline.run(stream,
index_host=index_host,
query_type=query_type,
dry_run=dry_run,
parallel=parallel)
return queries
def prepare_param(selection=None,path=None,parameter=None,load_default=None):
"""This func adds 'path', 'parameter' and 'selection' input type to the
standalone param pipeline.
Note
This func is used for light scenario (complex scenario use 'sdquerypipeline' module).
"""
if parameter is None:
parameter=[]
if selection is None:
buffer=sdbuffer.get_selection_file_buffer(path=path,parameter=parameter)
selection=sdparse.build(buffer,load_default=load_default)
facets_groups=selection.merge_facets()
facets_groups=sdparampipeline.run(facets_groups)
return facets_groups
def build_selection_list():
"""
Return:
selections list.
"""
selections = []
files = build_selection_file_list(
) # contains selection files path list (fullpath)
for file in files:
try:
buffer = sdbuffer.get_selection_file_buffer(path=file)
selection = sdparse.build(buffer)
selections.append(selection)
except Exception, e:
sdlog.error("SDSELGPU-001", "Exception occured (%s)" % str(e))
raise SDException(
"SDSELGPU-001",
"Error occured while loading '%s' selection file. See log for details."
% file)
def get_stream(parameter=None,selection_file=None,no_default=True,raise_exception_if_empty=False):
"""
TODO: merge me with syndautils.get_stream
"""
if parameter is None:
parameter=[]
buffer=sdbuffer.get_selection_file_buffer(parameter=parameter,path=selection_file)
if len(buffer.lines)==0:
# we come here when user selected nothing (neither from stdin neither from selection file neither from arg)
if raise_exception_if_empty:
raise sdexception.EmptySelectionException()
selection=sdparse.build(buffer,load_default=(not no_default))
stream=selection.to_stream()
return stream # aka facets_groups
def get_stream(parameter=None,
selection_file=None,
no_default=True,
raise_exception_if_empty=False):
"""
TODO: merge me with syndautils.get_stream
"""
if parameter is None:
parameter = []
buffer = sdbuffer.get_selection_file_buffer(parameter=parameter,
path=selection_file)
if len(buffer.lines) == 0:
# we come here when user selected nothing (neither from stdin neither from selection file neither from arg)
if raise_exception_if_empty:
raise sdexception.EmptySelectionException()
selection = sdparse.build(buffer, load_default=(not no_default))
stream = selection.to_stream()
return stream # aka facets_groups
def prepare_param(selection=None,
path=None,
parameter=None,
load_default=None):
"""This func adds 'path', 'parameter' and 'selection' input type to the
standalone param pipeline.
Note
This func is used for light scenario (complex scenario use 'sdquerypipeline' module).
"""
if parameter is None:
parameter = []
if selection is None:
buffer = sdbuffer.get_selection_file_buffer(path=path,
parameter=parameter)
selection = sdparse.build(buffer, load_default=load_default)
facets_groups = selection.merge_facets()
facets_groups = sdparampipeline.run(facets_groups)
return facets_groups
sdtiaction.actions[args.action](args)
elif args.action=='help':
parser.print_help()
elif args.action=='update':
print_stderr('Not implemented yet.')
elif args.action=='param':
import sdparam
sdparam.main(args.parameter) # tricks to re-use sdparam CLI parser
elif args.action in ['dump','install','list','remove','search','show','stat','version']:
import sdbuffer, sdparse, sdstream, sdconfig, sddeferredbefore
# hack
if args.action=='list':
args.no_default=True
buffer=sdbuffer.get_selection_file_buffer(parameter=args.parameter,path=args.selection)
selection=sdparse.build(buffer,load_default=(not args.no_default))
stream=selection.to_stream()
# Set default value for nearest here
#
# TODO: make it work with all actions (e.g. search) as it only working for 'install' action for now
#
#sddeferredbefore.add_default_parameter(stream,'nearest',True) # TODO: why this one is not working ?
if sdconfig.config.getboolean('behaviour','nearest'):
sdstream.set_scalar(stream,'nearest',True)
# progress
if sdconfig.config.getboolean('interface','progress'):
sdstream.set_scalar(stream,'progress',True)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""examples of use
cat file | %s
%s file
""" % (prog, prog))
parser.add_argument('file', nargs='?', default='-')
parser.add_argument('-1', '--print_only_one_item', action='store_true')
parser.add_argument('-F',
'--format',
choices=sdprint.formats,
default='raw')
parser.add_argument('-i', '--index_host')
parser.add_argument('--parallel', dest='parallel', action='store_true')
parser.add_argument('--no-parallel', dest='parallel', action='store_false')
parser.set_defaults(parallel=True)
args = parser.parse_args()
buffer = sdbuffer.get_selection_file_buffer(path=args.file)
selection = sdparse.build(buffer)
facets_groups = selection.merge_facets()
queries = run(facets_groups,
parallel=args.parallel,
index_host=args.index_host)
sdprint.print_format(queries, args.format, args.print_only_one_item)
def is_ffv_parameter(parameter): # ffv means 'Free Facets n Variable'
if parameter.count('[') == 1:
return True
else:
return False
# module init.
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('parameter', nargs='*', default=[], help=sdi18n.m0001)
parser.add_argument('-1', '--print_only_one_item', action='store_true')
parser.add_argument('-f', '--file', default=None)
parser.add_argument('-F',
'--format',
choices=sdprint.formats,
default='raw')
parser.add_argument('-n',
'--no_default',
action='store_true',
help='This option prevent loading default values')
args = parser.parse_args()
buffer = sdbuffer.get_selection_file_buffer(path=args.file,
parameter=args.parameter)
selection = build(buffer, load_default=(not args.no_default))
facets_groups = selection.merge_facets()
sdprint.print_format(facets_groups, args.format, args.print_only_one_item)
raise SDException("SDQUERYP-001","Unknow query type (%s)"%query_type)
return queries
if __name__ == '__main__':
prog=os.path.basename(__file__)
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""examples of use
cat file | %s
%s file
"""%(prog,prog))
parser.add_argument('file',nargs='?',default='-')
parser.add_argument('-1','--print_only_one_item',action='store_true')
parser.add_argument('-F','--format',choices=sdprint.formats,default='raw')
parser.add_argument('-i','--index_host')
parser.add_argument('--parallel',dest='parallel',action='store_true')
parser.add_argument('--no-parallel',dest='parallel',action='store_false')
parser.set_defaults(parallel=True)
args = parser.parse_args()
buffer=sdbuffer.get_selection_file_buffer(path=args.file)
selection=sdparse.build(buffer)
facets_groups=selection.merge_facets()
queries=run(facets_groups,parallel=args.parallel,index_host=args.index_host)
sdprint.print_format(queries,args.format,args.print_only_one_item)