if dry_run:
print url
return {}
# retrieve parameters
params=sdnetutils.call_param_web_service(url,60)
return params
# init.
if __name__ == '__main__':
prog=os.path.basename(__file__)
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog="""examples of use
%s
"""%sdcliex.search(prog))
parser.add_argument('parameter',nargs='*',default=[],help=sdi18n.m0001)
parser.add_argument('-H','--host',help='Index hostname')
parser.add_argument('-n','--name',help='Parameter name to be retrieve')
parser.add_argument('-y','--dry_run',action='store_true')
args = parser.parse_args()
import sdpipeline # BEWARE: keep it here because of circular dependency problem here: see TAG3434 tag in TODO file
facets_groups=sdpipeline.prepare_param(parameter=args.parameter)
facets_group=facets_groups[0]
params=run(pname=args.name,facets_group=facets_group,dry_run=args.dry_run,host=args.host)
if len(params)>0:
for name,values in params.iteritems():
for value in values:
datanode_place)
if distance < min_distance:
nearest_datanode = datanode
return nearest_datanode
if __name__ == '__main__':
prog = os.path.basename(__file__)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""examples of use
%s -d %s
%s
""" % (prog, 'esg-datanode.jpl.nasa.gov', sdcliex.search(prog)))
parser.add_argument('parameter', nargs='*', default=[], help=sdi18n.m0001)
parser.add_argument('-d', '--datanode', help='e.g. esgnode2.nci.org.au')
parser.add_argument('-s', '--show_candidate', action='store_true')
parser.add_argument('-y', '--dry_run', action='store_true')
args = parser.parse_args()
if args.datanode is not None:
# compute distance between us and the datanode
client_place = sdnearestutils.get_client_place() # our location
datanode_place = sdnearestutils.get_datanode_place(args.datanode)
distance = sdnearestutils.compute_distance(client_place,
datanode_place)
print "size: %s (%s)" % (f.size, humanize.naturalsize(f.size, gnu=False))
print "checksum: %s" % f.checksum
print "url: %s" % f.url
local_path_label = 'local path' if f.status in (
sdconst.TRANSFER_STATUS_DELETE,
sdconst.TRANSFER_STATUS_DONE) else 'local path (once downloaded)'
print "%s: %s" % (local_path_label, f.get_full_local_path())
print "replica: %s" % f.replica
print "data_node: %s" % f.data_node
# init.
if __name__ == '__main__':
prog = os.path.basename(__file__)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""examples of use
%s
""" % sdcliex.search(prog))
parser.add_argument('parameter', nargs='+', help=sdi18n.m0001)
parser.add_argument('-y', '--dry_run', action='store_true')
args = parser.parse_args()
files = get_files(parameter=args.parameter, dry_run=args.dry_run)
if not args.dry_run:
print_list(files)
# MEMO: when action is 'install', type is always 'File' (i.e. this code gets executed only for type=File)
#
if action is not None:
if action=='install':
metadata=sdadddsattr.run(squeries,metadata,parallel)
return metadata
if __name__ == '__main__':
prog=os.path.basename(__file__)
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""examples of use
%s -f file
cat file | %s
%s
"""%(prog,prog,sdcliex.search(prog)))
parser.add_argument('parameter',nargs='*',default=[],help=sdi18n.m0001)
parser.add_argument('-f','--file',default=None)
parser.add_argument('-F','--format',choices=sdprint.formats,default='raw')
parser.add_argument('-i','--index_host')
parser.add_argument('-m','--post_pipeline_mode',default='file',choices=sdconst.POST_PIPELINE_MODES)
parser.add_argument('-y','--dry_run',action='store_true')
parser.add_argument('-1','--print_only_one_item',action='store_true')
sdcommonarg.add_playback_record_options(parser)
parser.add_argument('--load-default',dest='load_default',action='store_true')
parser.add_argument('--no-load-default',dest='load_default',action='store_false')
parser.set_defaults(load_default=None)
def run(subparsers):
subparser = create_subparser(subparsers,
'autoremove',
selection=False,
no_default=False,
help='Remove old datasets versions')
subparser = create_subparser(subparsers,
'certificate',
common_option=False,
help='Manage X509 certificate',
example=sdcliex.certificate())
add_action_argument(subparser, choices=['renew', 'info', 'print'])
subparser.add_argument('-d',
'--debug',
action='store_true',
help='Display debug message')
subparser.add_argument('-o', '--openid', help='ESGF openid')
subparser.add_argument('-p', '--password', help='ESGF password')
subparser.add_argument('-x',
'--force_renew_ca_certificates',
action='store_true',
help='Force renew CA certificates')
subparser = create_subparser(subparsers,
'check',
no_default=False,
help='Perform check over ESGF metadata',
example=sdcliex.check(),
description=sddescription.check())
sdcommonarg.add_playback_record_options(subparser)
add_action_argument(
subparser, choices=['dataset_version', 'file_variable', 'selection'])
add_parameter_argument(subparser)
subparser.add_argument('-F',
'--output_format',
help='Set output format',
default='text',
choices=['text', 'pdf'])
subparser.add_argument('-o',
'--outfile',
default='/tmp/dataset_version_report.pdf')
subparser = create_subparser(subparsers,
'config',
common_option=False,
help='Print configuration information',
example=sdcliex.config())
subparser.add_argument(
'-n',
'--name',
default=None,
help=
'Name of the parameter to be displayed (if not set, all parameters are displayed)'
)
subparser = create_subparser(subparsers,
'contact',
common_option=False,
help='Print contact information')
subparser = create_subparser(subparsers,
'count',
help='Count file / dataset',
example=sdcliex.count())
subparser.add_argument('-i',
'--index_host',
help='Retrieve parameters from the specified index')
add_parameter_argument(subparser)
add_timestamp_boundaries(subparser, hidden=False)
sdcommonarg.add_type_grp(subparser)
subparser = create_subparser(subparsers,
'daemon',
common_option=False,
help='Daemon management',
note=sdi18n.m0023)
add_action_argument(subparser, choices=['start', 'stop', 'status'])
subparser = create_subparser(subparsers,
'dump',
help='Display raw metadata',
example=sdcliex.dump())
add_parameter_argument(subparser)
sdcommonarg.add_type_grp(subparser)
add_dump_option(subparser)
subparser = create_subparser(subparsers,
'facet',
no_default=False,
help='Facet discovery',
example=sdcliex.facet())
subparser.add_argument('facet_name', help='Facet name')
add_parameter_argument(subparser)
subparser = create_subparser(subparsers,
'get',
no_default=False,
help='Download dataset (sync)',
example=sdcliex.get())
subparser.add_argument('--verify_checksum',
'-c',
action='store_true',
help='Compare remote and local checksum')
subparser.add_argument(
'--dest_folder',
'-d',
default=sdconfig.files_dest_folder_for_get_subcommand,
help='Destination folder')
subparser.add_argument('--force',
'-f',
action='store_true',
help='Overwrite local file if exists')
subparser.add_argument(
'--network_bandwidth_test',
'-n',
action='store_true',
help=
'Prevent disk I/O to measure network throughput. When this option is used, local file is set to /dev/null.'
)
subparser.add_argument('--openid', '-o', help='ESGF openid')
subparser.add_argument('--password', '-p', help='ESGF password')
subparser.add_argument('--quiet', '-q', action='store_true')
subparser.add_argument('--timeout',
'-t',
type=int,
default=sdconst.DIRECT_DOWNLOAD_HTTP_TIMEOUT,
help='HTTP timeout')
subparser.add_argument('--urllib2',
'-u',
action='store_true',
help='Use urllib2 instead of wget as HTTP client')
subparser.add_argument(
'--verbosity', '-v', action='count', default=1
) # TAG43534FSFS. As default verbosity is 1, the only way to disable verbosity (i.e. set it to 0) is to use '--quiet' option. Also note that as default is 1 for an argparse 'count' option, '-v' triggers the value 2, not 1, and '-vv' triggers the value 3, not 2: this is normal. To summarize, what we do here is map 'unset' to 1, map [-vvvvvvv...] to the range [2-N], and add '--quiet' option to trigger verbosity level 0.
#
subparser.add_argument('--hpss',
dest='hpss',
action='store_true',
help="Enable 'hpss' flag")
subparser.add_argument('--no-hpss',
dest='hpss',
action='store_false',
help="Disable 'hpss' flag (Default)")
subparser.set_defaults(
hpss=False
) # maybe use sdconfig.config.getboolean('download','hpss') as default
#
add_parameter_argument(subparser)
subparser = subparsers.add_parser('help', help='Show help')
subparser.add_argument('topic', nargs='?')
subparser = create_subparser(subparsers,
'check-env',
help='Checks install environment.')
subparser = create_subparser(subparsers,
'init-env',
help='Initilizes install environment.')
subparser = create_subparser(subparsers,
'history',
common_option=False,
help='Show history')
subparser = create_subparser(subparsers,
'install',
help='Download dataset (async)',
note=sdi18n.m0022,
example=sdcliex.install())
add_ni_option(subparser)
add_parameter_argument(subparser)
add_incremental_mode_argument(subparser, 'install')
add_timestamp_boundaries(
subparser, hidden=True) # hidden option mainly used for test and debug
sdcommonarg.add_playback_record_options(subparser, hidden=False)
subparser = create_subparser(subparsers,
'intro',
common_option=False,
help='Print introduction to synda command')
subparser = create_subparser(
subparsers,
'list',
no_default=False,
help='List installed dataset',
example=sdcliex.list()
) # here no_default is a flag to decide if we show no_default option or not
subparser.set_defaults(
no_default=True) # here no_default is the real option
add_parameter_argument(subparser)
sdcommonarg.add_type_grp(subparser)
subparser = create_subparser(
subparsers,
'metric',
selection=False,
no_default=False,
help='Display performance and disk usage metrics',
example=sdcliex.metric())
subparser.add_argument('--groupby',
'-g',
choices=['data_node', 'project', 'model'],
default='data_node',
help='Group-by clause')
subparser.add_argument('--metric',
'-m',
choices=['rate', 'size'],
default='rate',
help='Metric name')
subparser.add_argument(
'--project',
'-p',
default='CMIP5',
help="Project name (must be used with '--groupby=model' else ignored)")
subparser = create_subparser(subparsers,
'open',
no_default=False,
help='Open netcdf file',
example=sdcliex.open())
subparser.add_argument('--geometry',
'-g',
default='1200x700+0+0',
help='Window geometry')
add_parameter_argument(subparser)
subparser = create_subparser(subparsers,
'param',
common_option=False,
help='Print ESGF facets',
example=sdcliex.param())
subparser.add_argument('pattern1',
nargs='?',
default=None,
help='Parameter name')
subparser.add_argument('pattern2', nargs='?', default=None, help='Filter')
subparser.add_argument('-c', '--columns', type=int, default=1)
subparser = create_subparser(subparsers,
'pexec',
help='Execute post-processing task')
subparser.add_argument('order_name',
choices=['cdf', 'cds'],
help='Order name')
subparser = create_subparser(subparsers,
'queue',
common_option=False,
help='Display download queue status',
example=sdcliex.queue())
subparser.add_argument('project',
nargs='?',
default=None,
help='ESGF project (e.g. CMIP5)')
subparser = create_subparser(subparsers,
'remove',
help='Remove dataset',
example=sdcliex.remove())
add_parameter_argument(subparser)
add_ni_option(subparser)
add_verbose_option(subparser)
subparser.add_argument('-m',
'--keep_data',
action='store_true',
help='Remove only metadata')
subparser = create_subparser(subparsers,
'replica',
selection=False,
no_default=False,
help='Move to next replica',
example=sdcliex.replica())
add_action_argument(subparser, choices=['next'])
subparser.add_argument('file_id',
nargs='?',
help='File identifier (ESGF instance_id)')
subparser = create_subparser(
subparsers,
'reset',
common_option=False,
help="Remove all 'waiting' and 'error' transfers")
subparser = create_subparser(
subparsers,
'retry',
common_option=False,
help='Retry transfer (switch status from error to waiting)')
subparser = create_subparser(subparsers,
'search',
help='Search dataset',
example=sdcliex.search('synda search'))
add_parameter_argument(subparser)
subparser.add_argument(
'-e', '--explode', action='store_true', help=argparse.SUPPRESS
) # explode id into individual facets (hidden option mainly used for debug)
subparser.add_argument('-l',
'--limit',
type=int,
default=sdconfig.get_default_limit('search'),
help=sdi18n.m0024)
subparser.add_argument('-r',
'--replica',
action='store_true',
help='show replica')
add_timestamp_boundaries(subparser, hidden=False)
sdcommonarg.add_type_grp(subparser)
subparser = create_subparser(subparsers,
'selection',
common_option=False,
help='List selection files')
subparser = create_subparser(
subparsers,
'show',
help='Display detailed information about dataset',
example=sdcliex.show())
add_parameter_argument(subparser)
add_lsearch_option(subparser)
#sdcommonarg.add_type_grp(subparser) # disabled as type depend on user input (e.g. file_functional_id, dataset_functional_id, etc..)
add_verbose_option(subparser)
subparser = create_subparser(
subparsers,
'stat',
help='Display summary information about dataset',
example=sdcliex.stat())
add_parameter_argument(subparser)
add_incremental_mode_argument(subparser, 'stat')
add_timestamp_boundaries(
subparser, hidden=True) # hidden option mainly used for test and debug
subparser = create_subparser(subparsers,
'token',
common_option=False,
help='Get OAuth2 tokens')
subparser.add_argument('-p',
'--provider',
choices=['globus'],
default='globus',
help='Get Globus OAuth2 tokens')
add_action_argument(subparser, choices=['renew', 'print'])
subparser = create_subparser(subparsers,
'update',
common_option=False,
help='Update ESGF parameter local cache')
subparser.add_argument('-i',
'--index_host',
help='Retrieve parameters from the specified index')
subparser.add_argument(
'-p',
'--project',
help='Retrieve project specific parameters for the specified project')
subparser = create_subparser(
subparsers,
'upgrade',
selection=False,
no_default=False,
help="Run 'install' command on all selection files")
add_parameter_argument(subparser)
add_ni_option(subparser)
add_incremental_mode_argument(subparser, 'upgrade')
add_timestamp_boundaries(
subparser, hidden=True) # hidden option mainly used for test and debug
subparser.add_argument('-e',
'--exclude_from',
metavar='FILE',
help='Read exclude selection-file from FILE')
subparser = create_subparser(subparsers,
'variable',
selection=False,
no_default=False,
help='Print variable',
example=sdcliex.variable())
add_parameter_argument(subparser)
subparser.add_argument('-l', '--long_name', action='store_true')
subparser.add_argument('-s', '--short_name', action='store_true')
subparser.add_argument('-S', '--standard_name', action='store_true')
subparser = create_subparser(subparsers,
'version',
help='List all versions of a dataset',
example=sdcliex.version())
add_parameter_argument(subparser)
#sdcommonarg.add_type_grp(subparser) # disabled as type depend on user input (e.g. file_functional_id, dataset_functional_id, etc..)
subparser = create_subparser(subparsers,
'watch',
common_option=False,
help='Display running transfer')
def run(subparsers):
subparser=create_subparser(subparsers,'autoremove',selection=False,no_default=False,help='Remove old datasets versions')
subparser=create_subparser(subparsers,'certificate',common_option=False,help='Manage X509 certificate',example=sdcliex.certificate())
add_action_argument(subparser,choices=['renew','info','print'])
subparser.add_argument('-d','--debug',action='store_true',help='Display debug message')
subparser.add_argument('-o','--openid',help='ESGF openid')
subparser.add_argument('-p','--password',help='ESGF password')
subparser.add_argument('-x','--force_renew_ca_certificates',action='store_true',help='Force renew CA certificates')
subparser=create_subparser(subparsers,'check',no_default=False,help='Perform check over ESGF metadata',example=sdcliex.check(),description=sddescription.check())
sdcommonarg.add_playback_record_options(subparser)
add_action_argument(subparser,choices=['dataset_version','file_variable','selection'])
add_parameter_argument(subparser)
subparser.add_argument('-F','--output_format',help='Set output format',default='text',choices=['text','pdf'])
subparser.add_argument('-o','--outfile',default='/tmp/dataset_version_report.pdf')
subparser=create_subparser(subparsers,'config',common_option=False,help='Print configuration information',example=sdcliex.config())
subparser.add_argument('-n','--name',default=None,help='Name of the parameter to be displayed (if not set, all parameters are displayed)')
subparser=create_subparser(subparsers,'contact',common_option=False,help='Print contact information')
subparser=create_subparser(subparsers,'count',help='Count file / dataset',example=sdcliex.count())
subparser.add_argument('-i','--index_host',help='Retrieve parameters from the specified index')
add_parameter_argument(subparser)
add_timestamp_boundaries(subparser,hidden=False)
sdcommonarg.add_type_grp(subparser)
subparser=create_subparser(subparsers,'daemon',common_option=False,help='Daemon management',note=sdi18n.m0023)
add_action_argument(subparser,choices=['start','stop','status'])
subparser=create_subparser(subparsers,'dump',help='Display raw metadata',example=sdcliex.dump())
add_parameter_argument(subparser)
sdcommonarg.add_type_grp(subparser)
add_dump_option(subparser)
subparser=create_subparser(subparsers,'facet',no_default=False,help='Facet discovery',example=sdcliex.facet())
subparser.add_argument('facet_name',help='Facet name')
add_parameter_argument(subparser)
subparser=create_subparser(subparsers,'get',no_default=False,help='Download dataset (sync)',example=sdcliex.get())
subparser.add_argument('--verify_checksum','-c',action='store_true',help='Compare remote and local checksum')
subparser.add_argument('--dest_folder','-d',default=sdconfig.files_dest_folder_for_get_subcommand,help='Destination folder')
subparser.add_argument('--force','-f',action='store_true',help='Overwrite local file if exists')
subparser.add_argument('--network_bandwidth_test','-n',action='store_true',help='Prevent disk I/O to measure network throughput. When this option is used, local file is set to /dev/null.')
subparser.add_argument('--openid','-o',help='ESGF openid')
subparser.add_argument('--password','-p',help='ESGF password')
subparser.add_argument('--quiet','-q', action='store_true')
subparser.add_argument('--timeout','-t',type=int,default=sdconst.DIRECT_DOWNLOAD_HTTP_TIMEOUT,help='HTTP timeout')
subparser.add_argument('--urllib2','-u',action='store_true',help='Use urllib2 instead of wget as HTTP client')
subparser.add_argument('--verbosity','-v', action='count', default=1) # TAG43534FSFS. As default verbosity is 1, the only way to disable verbosity (i.e. set it to 0) is to use '--quiet' option. Also note that as default is 1 for an argparse 'count' option, '-v' triggers the value 2, not 1, and '-vv' triggers the value 3, not 2: this is normal. To summarize, what we do here is map 'unset' to 1, map [-vvvvvvv...] to the range [2-N], and add '--quiet' option to trigger verbosity level 0.
#
subparser.add_argument('--hpss',dest='hpss',action='store_true',help="Enable 'hpss' flag")
subparser.add_argument('--no-hpss',dest='hpss',action='store_false',help="Disable 'hpss' flag (Default)")
subparser.set_defaults(hpss=False) # maybe use sdconfig.config.getboolean('download','hpss') as default
#
add_parameter_argument(subparser)
subparser=subparsers.add_parser('help',help='Show help')
subparser.add_argument('topic',nargs='?')
subparser=create_subparser(subparsers,'history',common_option=False,help='Show history')
subparser=create_subparser(subparsers,'install',help='Download dataset (async)',note=sdi18n.m0022,example=sdcliex.install())
add_ni_option(subparser)
add_parameter_argument(subparser)
add_incremental_mode_argument(subparser,'install')
add_timestamp_boundaries(subparser,hidden=True) # hidden option mainly used for test and debug
sdcommonarg.add_playback_record_options(subparser,hidden=False)
subparser=create_subparser(subparsers,'intro',common_option=False,help='Print introduction to synda command')
subparser=create_subparser(subparsers,'list',no_default=False,help='List installed dataset',example=sdcliex.list()) # here no_default is a flag to decide if we show no_default option or not
subparser.set_defaults(no_default=True) # here no_default is the real option
add_parameter_argument(subparser)
sdcommonarg.add_type_grp(subparser)
subparser=create_subparser(subparsers,'metric',selection=False,no_default=False,help='Display performance and disk usage metrics',example=sdcliex.metric())
subparser.add_argument('--groupby','-g',choices=['data_node','project','model'],default='data_node',help='Group-by clause')
subparser.add_argument('--metric','-m',choices=['rate','size'],default='rate',help='Metric name')
subparser.add_argument('--project','-p',default='CMIP5',help="Project name (must be used with '--groupby=model' else ignored)")
subparser=create_subparser(subparsers,'open',no_default=False,help='Open netcdf file',example=sdcliex.open())
subparser.add_argument('--geometry','-g',default='1200x700+0+0',help='Window geometry')
add_parameter_argument(subparser)
subparser=create_subparser(subparsers,'param',common_option=False,help='Print ESGF facets',example=sdcliex.param())
subparser.add_argument('pattern1',nargs='?',default=None,help='Parameter name')
subparser.add_argument('pattern2',nargs='?',default=None,help='Filter')
subparser.add_argument('-c','--columns',type=int,default=1)
subparser=create_subparser(subparsers,'pexec',help='Execute post-processing task')
subparser.add_argument('order_name',choices=['cdf','cds'],help='Order name')
subparser=create_subparser(subparsers,'queue',common_option=False,help='Display download queue status',example=sdcliex.queue())
subparser.add_argument('project',nargs='?',default=None,help='ESGF project (e.g. CMIP5)')
subparser=create_subparser(subparsers,'remove',help='Remove dataset',example=sdcliex.remove())
add_parameter_argument(subparser)
add_ni_option(subparser)
add_verbose_option(subparser)
subparser.add_argument('-m','--keep_data',action='store_true',help='Remove only metadata')
subparser=create_subparser(subparsers,'replica',selection=False,no_default=False,help='Move to next replica',example=sdcliex.replica())
add_action_argument(subparser,choices=['next'])
subparser.add_argument('file_id',nargs='?',help='File identifier (ESGF instance_id)')
subparser=create_subparser(subparsers,'reset',common_option=False,help="Remove all 'waiting' and 'error' transfers")
subparser=create_subparser(subparsers,'retry',common_option=False,help='Retry transfer (switch status from error to waiting)')
subparser=create_subparser(subparsers,'search',help='Search dataset',example=sdcliex.search('synda search'))
add_parameter_argument(subparser)
subparser.add_argument('-e','--explode',action='store_true',help=argparse.SUPPRESS) # explode id into individual facets (hidden option mainly used for debug)
subparser.add_argument('-l','--limit',type=int,default=sdconfig.get_default_limit('search'),help=sdi18n.m0024)
subparser.add_argument('-r','--replica',action='store_true',help='show replica')
add_timestamp_boundaries(subparser,hidden=False)
sdcommonarg.add_type_grp(subparser)
subparser=create_subparser(subparsers,'selection',common_option=False,help='List selection files')
subparser=create_subparser(subparsers,'show',help='Display detailed information about dataset',example=sdcliex.show())
add_parameter_argument(subparser)
add_lsearch_option(subparser)
#sdcommonarg.add_type_grp(subparser) # disabled as type depend on user input (e.g. file_functional_id, dataset_functional_id, etc..)
add_verbose_option(subparser)
subparser=create_subparser(subparsers,'stat',help='Display summary information about dataset',example=sdcliex.stat())
add_parameter_argument(subparser)
add_incremental_mode_argument(subparser,'stat')
add_timestamp_boundaries(subparser,hidden=True) # hidden option mainly used for test and debug
subparser=create_subparser(subparsers,'update',common_option=False,help='Update ESGF parameter local cache')
subparser.add_argument('-i','--index_host',help='Retrieve parameters from the specified index')
subparser.add_argument('-p','--project',help='Retrieve project specific parameters for the specified project')
subparser=create_subparser(subparsers,'upgrade',selection=False,no_default=False,help="Run 'install' command on all selection files")
add_parameter_argument(subparser)
add_ni_option(subparser)
add_incremental_mode_argument(subparser,'upgrade')
add_timestamp_boundaries(subparser,hidden=True) # hidden option mainly used for test and debug
subparser.add_argument('-e','--exclude_from',metavar='FILE',help='Read exclude selection-file from FILE')
subparser=create_subparser(subparsers,'variable',selection=False,no_default=False,help='Print variable',example=sdcliex.variable())
add_parameter_argument(subparser)
subparser.add_argument('-l','--long_name',action='store_true')
subparser.add_argument('-s','--short_name',action='store_true')
subparser.add_argument('-S','--standard_name',action='store_true')
subparser=create_subparser(subparsers,'version',help='List all versions of a dataset',example=sdcliex.version())
add_parameter_argument(subparser)
#sdcommonarg.add_type_grp(subparser) # disabled as type depend on user input (e.g. file_functional_id, dataset_functional_id, etc..)
subparser=create_subparser(subparsers,'watch',common_option=False,help='Display running transfer')
def run(subparsers):
subparser=create_subparser(subparsers,'autoremove',selection=False,no_default=False,help='Remove old datasets versions')
subparser=create_subparser(subparsers,'certificate',common_option=False,help='Manage X509 certificate',example=sdcliex.certificate())
add_action_argument(subparser,choices=['renew','print'])
subparser.add_argument('-x','--force_renew_ca_certificates',action='store_true',help='Force renew CA certificates')
subparser=create_subparser(subparsers,'contact',common_option=False,help='Print contact information')
subparser=create_subparser(subparsers,'daemon',common_option=False,help='Daemon management')
add_action_argument(subparser,choices=['start','stop','status'])
subparser=create_subparser(subparsers,'dump',help='Display raw metadata',example=sdcliex.dump())
add_parameter_argument(subparser)
add_type_grp(subparser)
add_dump_option(subparser)
subparser=create_subparser(subparsers,'facet',help='Facet discovery',example=sdcliex.facet())
subparser.add_argument('facet_name',help='Facet name')
add_parameter_argument(subparser)
subparser=subparsers.add_parser('help',help='Show help')
subparser.add_argument('topic',nargs='?')
subparser=create_subparser(subparsers,'history',common_option=False,help='Show history')
subparser=create_subparser(subparsers,'install',help='Install dataset')
add_ni_option(subparser)
add_parameter_argument(subparser)
subparser=create_subparser(subparsers,'intro',common_option=False,help='Print introduction to synda command')
subparser=create_subparser(subparsers,'list',help='List installed dataset',example=sdcliex.list())
add_parameter_argument(subparser)
add_type_grp(subparser)
subparser=create_subparser(subparsers,'metric',common_option=False,help='Display performance and disk usage metrics',example=sdcliex.metric())
subparser.add_argument('--groupby','-g',choices=['data_node','project','model'],default='data_node',help='Group-by clause')
subparser.add_argument('--metric','-m',choices=['rate','size'],default='rate',help='Metric name')
subparser.add_argument('--project','-p',default='CMIP5',help="Project name (must be used with '--groupby=model' else ignored)")
subparser=create_subparser(subparsers,'param',common_option=False,help='Display ESGF parameters',example=sdcliex.param())
subparser.add_argument('pattern1',nargs='?',default=None,help='Parameter name')
subparser.add_argument('pattern2',nargs='?',default=None,help='Filter')
subparser.add_argument('-c','--columns',type=int,default=1)
subparser=create_subparser(subparsers,'pexec',help='Execute post-processing task')
subparser.add_argument('order_name',help='Order name')
add_type_grp(subparser)
subparser=create_subparser(subparsers,'queue',common_option=False,help='Display download queue status',example=sdcliex.queue())
subparser.add_argument('project',nargs='?',default=None,help='ESGF project (e.g. CMIP5)')
subparser=create_subparser(subparsers,'remove',help='Remove dataset',example=sdcliex.remove())
add_parameter_argument(subparser)
subparser=create_subparser(subparsers,'replica',selection=False,no_default=False,help='Move to next replica',example=sdcliex.replica())
add_action_argument(subparser,choices=['next'])
subparser.add_argument('file_id',nargs='?',help='File identifier (ESGF instance_id)')
subparser=create_subparser(subparsers,'reset',common_option=False,help="Remove all 'waiting' and 'error' transfers")
subparser=create_subparser(subparsers,'retry',common_option=False,help='Retry transfer (switch error status to waiting)')
subparser=create_subparser(subparsers,'search',help='Search dataset',example=sdcliex.search('synda search'))
add_parameter_argument(subparser)
subparser.add_argument('-r','--replica',action='store_true',help='show replica')
add_type_grp(subparser)
subparser=create_subparser(subparsers,'selection',common_option=False,help='List selection files')
subparser=create_subparser(subparsers,'show',help='Display detailed information about dataset',example=sdcliex.show())
add_parameter_argument(subparser)
add_lsearch_option(subparser)
#add_type_grp(subparser) # disabled as type depend on user input (e.g. file_functional_id, dataset_functional_id, etc..)
add_verbose_option(subparser)
subparser=create_subparser(subparsers,'stat',help='Display summary information about dataset',example=sdcliex.stat())
add_parameter_argument(subparser)
subparser=create_subparser(subparsers,'test',common_option=False,help='Test file download',example=sdcliex.test())
subparser.add_argument('file_url',help='file url')
subparser=create_subparser(subparsers,'update',common_option=False,help='Update ESGF parameter local cache')
subparser.add_argument('-i','--index_host',help='Retrieve parameters from the specified index')
subparser.add_argument('-p','--project',help='Retrieve project specific parameters for the specified project')
subparser=create_subparser(subparsers,'upgrade',selection=False,no_default=False,help='Perform an upgrade (retrieve new version for all selection files)')
add_parameter_argument(subparser)
add_ni_option(subparser)
subparser=create_subparser(subparsers,'version',help='List all versions of a dataset',example=sdcliex.version())
add_parameter_argument(subparser)
#add_type_grp(subparser) # disabled as type depend on user input (e.g. file_functional_id, dataset_functional_id, etc..)
subparser=create_subparser(subparsers,'watch',common_option=False,help='Display running transfer')
return mqr
def ws_call(query):
request=sdtypes.Request(url=query['url'],pagination=False)
result=sdnetutils.call_web_service(request.get_url(),timeout=sdconst.SEARCH_API_HTTP_TIMEOUT) # return Response object
if result.count()>=sdconst.SEARCH_API_CHUNKSIZE:
raise SDException("SDQSEARC-002","Number of returned files reach maximum limit")
result=sdaddap.run(result,query.get('attached_parameters',{}))
return result
if __name__ == '__main__':
prog=os.path.basename(__file__)
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog="""examples of use\n%s"""%sdcliex.search(prog))
parser.add_argument('parameter',nargs='*',default=[],help=sdi18n.m0001)
parser.add_argument('-F','--format',choices=sdprint.formats,default='indent')
parser.add_argument('-i','--index_host')
parser.add_argument('-m','--post_pipeline_mode',default='file')
parser.add_argument('-y','--dry_run',action='store_true')
parser.add_argument('-1','--print_only_one_item',action='store_true')
args = parser.parse_args()
result=run(parameter=args.parameter,index_host=args.index_host,post_pipeline_mode=args.post_pipeline_mode,dry_run=args.dry_run)
sdprint.print_format(result.get_files(),args.format,args.print_only_one_item)
for datanode in datanodes:
datanode_place=sdnearestutils.get_datanode_place(datanode)
distance=sdnearestutils.compute_distance(client_place,datanode_place)
if distance<min_distance:
nearest_datanode=datanode
return nearest_datanode
if __name__ == '__main__':
prog=os.path.basename(__file__)
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog="""examples of use
%s -d %s
%s
"""%(prog,'esg-datanode.jpl.nasa.gov',sdcliex.search(prog)))
parser.add_argument('parameter',nargs='*',default=[],help=sdi18n.m0001)
parser.add_argument('-d', '--datanode',help='e.g. esgnode2.nci.org.au')
parser.add_argument('-s', '--show_candidate',action='store_true')
parser.add_argument('-y', '--dry_run',action='store_true')
args = parser.parse_args()
if args.datanode is not None:
# compute distance between us and the datanode
client_place=sdnearestutils.get_client_place() # our location
datanode_place=sdnearestutils.get_datanode_place(args.datanode)
distance=sdnearestutils.compute_distance(client_place,datanode_place)
print "The datanode is %i km from your location (%s)"%(distance,sdnearestutils.get_client_country())
timeout=sdconst.SEARCH_API_HTTP_TIMEOUT) # return Response object
if result.count() > sdconst.SEARCH_API_CHUNKSIZE:
raise SDException("SDQSEARC-002",
"Number of returned files reach maximum limit")
result = sdaddap.run(result, query.get('attached_parameters', {}))
return result
if __name__ == '__main__':
prog = os.path.basename(__file__)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""examples of use\n%s""" % sdcliex.search(prog))
parser.add_argument('parameter', nargs='*', default=[], help=sdi18n.m0001)
parser.add_argument('-i', '--index_host')
parser.add_argument('-z', '--dry_run', action='store_true')
sdcommonarg.add_type_grp(parser)
args = parser.parse_args()
result = run(parameter=args.parameter,
index_host=args.index_host,
dry_run=args.dry_run,
type_=args.type_)
def help_search(self):
print sdi18n.m0006('search [FILTER]... [ limit ]','Search file(s)',example=sdcliex.search('search'),note=sdi18n.m0005)
def help_search(self):
print sdi18n.m0006('search [FILTER]... [ limit ]',
'Search file(s)',
example=sdcliex.search('search'),
note=sdi18n.m0005)