def dataset_show(args):
import sdearlystreamutils
# check
li=sdearlystreamutils.get_facet_values_early(args.stream,'instance_id')
if len(li)==0:
print_stderr('Please specify a dataset name.')
return 1
elif len(li)>1:
print_stderr('Too many arguments.')
return 1
if args.localsearch:
import sdldataset
dataset=sdldataset.get_dataset(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if dataset is None:
print_stderr("Dataset not found")
else:
sdldataset.print_details(dataset)
else:
import sdrdataset
dataset=sdrdataset.get_dataset(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if dataset is None:
print_stderr("Dataset not found")
else:
sdrdataset.print_details(dataset,verbose=args.verbose)
def dataset_version(args):
import sdremoteparam, syndautils, sdearlystreamutils
# don't be misled about identifiers here: sdinference produces search-api
# key (always do) name i.e. instance_id, and I use Synda style variable name
# i.e. dataset_functional_id (for better readability).
li = sdearlystreamutils.get_facet_values_early(args.stream, 'instance_id')
if len(li) == 0:
print_stderr('Please specify a dataset name.')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
else:
dataset_functional_id = li[0]
dataset_functional_id_without_version = syndautils.strip_dataset_version(
dataset_functional_id)
params = sdremoteparam.run(pname='version',
facets_group={
'type': [sdconst.SA_TYPE_DATASET],
'master_id':
[dataset_functional_id_without_version]
},
dry_run=args.dry_run)
# TODO: func for code below
items = params.get('version', [])
for item in items:
print item.name
def dataset_show(args):
import sdearlystreamutils
# check
li = sdearlystreamutils.get_facet_values_early(args.stream, 'instance_id')
if len(li) == 0:
print_stderr('Please specify a dataset name.')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
if args.localsearch:
import sdldataset
dataset = sdldataset.get_dataset(stream=args.stream,
dry_run=args.dry_run)
if not args.dry_run:
if dataset is None:
print_stderr("Dataset not found")
else:
sdldataset.print_details(dataset)
else:
import sdrdataset
dataset = sdrdataset.get_dataset(stream=args.stream,
dry_run=args.dry_run)
if not args.dry_run:
if dataset is None:
print_stderr("Dataset not found")
else:
sdrdataset.print_details(dataset, verbose=args.verbose)
def get_projects(selection):
"""This method returns project (standalone pending, standalone non-pending
or as part of an identifier) from CLI parameter or selection file."""
project = []
# project with keyname
if 'project' in selection.facets:
project += selection.facets['project']
# project without keyname
# WARNING
#
# The code below uses sdinference and query the database to retrieve ESGF parameters.
# Doing those things here may raise circular dependencies
# as well as making the whole thing very complex.
#
# We do this to make this syntax work (i.e. project value without key)
# synda search GeoMIP
#
# Note that this syntax always works (i.e. load the project level default file), even without this code.
# synda search project=GeoMIP
#
#
pending_projects = sdearlystreamutils.get_facet_values_early(
[selection.facets], 'project', extract_item=True
) # project without keyname or project as part of an identifier.
li = pending_projects + project
li = list(set(li)) # remove duplicate
return li
def get_projects(selection):
"""This method returns project (standalone pending, standalone non-pending
or as part of an identifier) from CLI parameter or selection file."""
project=[]
# project with keyname
if 'project' in selection.facets:
project+=selection.facets['project']
# project without keyname
# WARNING
#
# The code below uses sdinference and query the database to retrieve ESGF parameters.
# Doing those things here may raise circular dependencies
# as well as making the whole thing very complex.
#
# We do this to make this syntax work (i.e. project value without key)
# synda search GeoMIP
#
# Note that this syntax always works (i.e. load the project level default file), even without this code.
# synda search project=GeoMIP
#
#
pending_projects=sdearlystreamutils.get_facet_values_early([selection.facets],'project',extract_item=True) # project without keyname or project as part of an identifier.
li=pending_projects+project
li=list(set(li)) # remove duplicate
return li
def file_show(args):
import sdearlystreamutils
# check
li = sdearlystreamutils.get_facet_values_early(
args.stream, 'instance_id') # check if 'instance_id' exists
if len(li) == 0:
# 'instance_id' is not found on cli
li = sdearlystreamutils.get_facet_values_early(
args.stream, 'title') # check if 'title' exists
if len(li) == 0:
# 'title' is not found on cli
# no identifier found, we stop the processing
print_stderr('Please specify a file identifier (id or filename).')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
# main
if args.localsearch:
import sdlfile
file = sdlfile.get_file(stream=args.stream, dry_run=args.dry_run)
if not args.dry_run:
if file is None:
print_stderr("File not found")
else:
sdlfile.print_details(file)
else:
import sdrfile
file = sdrfile.get_file(stream=args.stream, dry_run=args.dry_run)
if not args.dry_run:
if file is None:
print_stderr("File not found")
else:
sdrfile.print_details(file)
def file_show(args):
import sdearlystreamutils
# check
li=sdearlystreamutils.get_facet_values_early(args.stream,'instance_id') # check if 'instance_id' exists
if len(li)==0:
# 'instance_id' is not found on cli
li=sdearlystreamutils.get_facet_values_early(args.stream,'title') # check if 'title' exists
if len(li)==0:
# 'title' is not found on cli
# no identifier found, we stop the processing
print_stderr('Please specify a file identifier (id or filename).')
return 1
elif len(li)>1:
print_stderr('Too many arguments.')
return 1
elif len(li)>1:
print_stderr('Too many arguments.')
return 1
# main
if args.localsearch:
import sdlfile
file=sdlfile.get_file(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if file is None:
print_stderr("File not found")
else:
sdlfile.print_details(file)
else:
import sdrfile
file=sdrfile.get_file(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if file is None:
print_stderr("File not found")
else:
sdrfile.print_details(file)
def apache_default_listing(args):
import sdearlystreamutils, sdhtmlbasic
urls = sdearlystreamutils.get_facet_values_early(args.stream, 'url')
if len(urls) == 0:
# no url in stream
print_stderr("Incorrect argument: please specify an url")
return 1
else:
files = sdhtmlbasic.get_files(stream=args.stream, dry_run=args.dry_run)
if not args.dry_run:
if len(files) == 0:
print_stderr("File not found")
else:
sdhtmlbasic.print_list(files)
def dataset_version(args):
import sdremoteparam,syndautils,sdearlystreamutils
# don't be misled about identifiers here: sdinference produces search-api
# key (always do) name i.e. instance_id, and I use Synda style variable name
# i.e. dataset_functional_id (for better readability).
li=sdearlystreamutils.get_facet_values_early(args.stream,'instance_id')
if len(li)==0:
print_stderr('Please specify a dataset name.')
return 1
elif len(li)>1:
print_stderr('Too many arguments.')
return 1
else:
dataset_functional_id=li[0]
dataset_functional_id_without_version=syndautils.strip_dataset_version(dataset_functional_id)
params=sdremoteparam.run(pname='version',facets_group={'type':[sdconst.SA_TYPE_DATASET],'master_id':[dataset_functional_id_without_version]},dry_run=args.dry_run)
# TODO: func for code below
items=params.get('version',[])
for item in items:
print item.name
def open_(args):
import sdview,syndautils,sdsandbox,sdtypes,sdconst,sdearlystreamutils
stream=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file)
# check
li=sdearlystreamutils.get_facet_values_early(stream,'instance_id') # check if 'instance_id' exists
if len(li)==0:
# 'instance_id' is not found on cli
li=sdearlystreamutils.get_facet_values_early(stream,'title') # check if 'title' exists
if len(li)==0:
# 'title' is not found on cli
# no identifier found, we stop the processing
print_stderr('Please specify a file identifier (id or filename).')
return 1
elif len(li)>1:
print_stderr('Too many arguments.')
return 1
elif len(li)>1:
print_stderr('Too many arguments.')
return 1
# discovery
import sdlfile
file_=sdlfile.get_file(stream=stream)
if file_ is None:
import sdrfile
file_=sdrfile.get_file(stream=stream)
if file_ is None:
print_stderr("File not found")
return 2
# cast
f=sdtypes.File(**file_)
# check if file exists locally
if f.status==sdconst.TRANSFER_STATUS_DONE:
file_local_path=f.get_full_local_path()
elif sdsandbox.file_exists(f.filename):
file_local_path=sdsandbox.get_file_path(f.filename)
else:
file_local_path=None
# download (if not done already)
if file_local_path is None:
status=sddirectdownload.run([file_], verbosity=1)
if status!=0:
return 1
# open file in external viewer
sdview.open_(file_local_path,f.variable,args.geometry)
return 0
def get(args):
import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils
# hack
# see TAG43534FSFS
if args.quiet:
args.verbosity=0
if args.verify_checksum and args.network_bandwidth_test:
print_stderr("'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set.")
return 1
stream=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file)
if args.openid and args.password:
# use credential from CLI
oid=args.openid
pwd=args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid=sdconfig.openid
pwd=sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).'%sdconfig.credential_file)
return 1
# retrieve certificate
sdlogon.renew_certificate(oid,pwd,force_renew_certificate=False)
http_client=sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET
# local_path
#
# 'synda get' subcommand currently force local_path to the following construct:
# '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may
# change in the future.
#
if args.dest_folder is None:
local_path_prefix=os.getcwd() # current working directory
else:
local_path_prefix=args.dest_folder
# BEWARE
#
# when set in CLI parameter, url is usually an ESGF facet, and as so should
# be sent to the search-api as other facets
# BUT
# we want a special behaviour here (i.e. with 'synda get' command) with url:
# if url is set by user, we DON'T call search-api operator. Instead, we
# download the url directly.
urls=sdearlystreamutils.get_facet_values_early(stream,'url')
if len(urls)==0:
# no url in stream: switch to search-api operator mode
sddeferredafter.add_default_parameter(stream,'limit',5)
sddeferredafter.add_forced_parameter(stream,'local_path_format','notree')
files=sdrfile.get_files(stream=stream,post_pipeline_mode='file',dry_run=args.dry_run) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent
if not args.dry_run:
if len(files)>0:
# compute metric
total_size=sum(int(f['size']) for f in files)
total_size=humanize.naturalsize(total_size,gnu=False)
print_stderr('%i file(s) will be downloaded for a total size of %s.'%(len(files),total_size))
status=sddirectdownload.run(files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum,
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status!=0:
return 1
else:
print_stderr("File not found")
return 1
else:
for f in files:
size=humanize.naturalsize(f['size'],gnu=False)
print '%-12s %s'%(size,f['filename'])
elif len(urls)>0:
# url(s) found in stream: search-api operator not needed (download url directly)
# TAGDSFDF432F
if args.verify_checksum:
print_stderr("To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url.")
return 1
# TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url))
files=[]
for url in urls:
filename=os.path.basename(url)
local_path=filename
f=dict(local_path=local_path,url=url)
files.append(f)
status=sddirectdownload.run(files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status!=0:
return 1
else:
assert False
return 0
def open_(args):
import sdview, syndautils, sdsandbox, sdtypes, sdconst, sdearlystreamutils
stream = syndautils.get_stream(subcommand=args.subcommand,
parameter=args.parameter,
selection_file=args.selection_file)
# check
li = sdearlystreamutils.get_facet_values_early(
stream, 'instance_id') # check if 'instance_id' exists
if len(li) == 0:
# 'instance_id' is not found on cli
li = sdearlystreamutils.get_facet_values_early(
stream, 'title') # check if 'title' exists
if len(li) == 0:
# 'title' is not found on cli
# no identifier found, we stop the processing
print_stderr('Please specify a file identifier (id or filename).')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
elif len(li) > 1:
print_stderr('Too many arguments.')
return 1
# discovery
import sdlfile
file_ = sdlfile.get_file(stream=stream)
if file_ is None:
import sdrfile
file_ = sdrfile.get_file(stream=stream)
if file_ is None:
print_stderr("File not found")
return 2
# cast
f = sdtypes.File(**file_)
# check if file exists locally
if f.status == sdconst.TRANSFER_STATUS_DONE:
file_local_path = f.get_full_local_path()
elif sdsandbox.file_exists(f.filename):
file_local_path = sdsandbox.get_file_path(f.filename)
else:
file_local_path = None
# download (if not done already)
if file_local_path is None:
status = sddirectdownload.run([file_], verbosity=1)
if status != 0:
return 1
# open file in external viewer
sdview.open_(file_local_path, f.variable, args.geometry)
return 0
def get(args):
import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils
# hack
# see TAG43534FSFS
if args.quiet:
args.verbosity = 0
if args.verify_checksum and args.network_bandwidth_test:
print_stderr(
"'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set."
)
return 1
stream = syndautils.get_stream(subcommand=args.subcommand,
parameter=args.parameter,
selection_file=args.selection_file)
if args.openid and args.password:
# use credential from CLI
oid = args.openid
pwd = args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid = sdconfig.openid
pwd = sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).' %
sdconfig.credential_file)
return 1
# retrieve certificate
sdlogon.renew_certificate(oid, pwd, force_renew_certificate=False)
http_client = sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET
# local_path
#
# 'synda get' subcommand currently force local_path to the following construct:
# '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may
# change in the future.
#
if args.dest_folder is None:
local_path_prefix = os.getcwd() # current working directory
else:
local_path_prefix = args.dest_folder
# BEWARE
#
# when set in CLI parameter, url is usually an ESGF facet, and as so should
# be sent to the search-api as other facets
# BUT
# we want a special behaviour here (i.e. with 'synda get' command) with url:
# if url is set by user, we DON'T call search-api operator. Instead, we
# download the url directly.
urls = sdearlystreamutils.get_facet_values_early(stream, 'url')
if len(urls) == 0:
# no url in stream: switch to search-api operator mode
sddeferredafter.add_default_parameter(stream, 'limit', 5)
sddeferredafter.add_forced_parameter(stream, 'local_path_format',
'notree')
files = sdrfile.get_files(
stream=stream, post_pipeline_mode='file', dry_run=args.dry_run
) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent
if not args.dry_run:
if len(files) > 0:
# compute metric
total_size = sum(int(f['size']) for f in files)
total_size = humanize.naturalsize(total_size, gnu=False)
print_stderr(
'%i file(s) will be downloaded for a total size of %s.' %
(len(files), total_size))
status = sddirectdownload.run(
files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum,
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status != 0:
return 1
else:
print_stderr("File not found")
return 1
else:
for f in files:
size = humanize.naturalsize(f['size'], gnu=False)
print '%-12s %s' % (size, f['filename'])
elif len(urls) > 0:
# url(s) found in stream: search-api operator not needed (download url directly)
# TAGDSFDF432F
if args.verify_checksum:
print_stderr(
"To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url."
)
return 1
# TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url))
files = []
for url in urls:
filename = os.path.basename(url)
local_path = filename
f = dict(local_path=local_path, url=url)
files.append(f)
status = sddirectdownload.run(
files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status != 0:
return 1
else:
assert False
return 0
