def variable_list(args):
import sddeferredafter
sddeferredafter.add_default_parameter(args.stream,'limit',15) # note: in variable mode, total number of row is given by: "total+=#variable for each ds"
print_stderr('Not implemented yet.')
"""
def file_dump(args):
import sdrfile, sddeferredafter, sdcolumnfilter, sdreducecol
sddeferredafter.add_default_parameter(args.stream,'limit',100)
if args.raw_mode:
post_pipeline_mode=None
args.all=True # we force '--all' option when '--raw_mode' option is set
else:
post_pipeline_mode='file'
files=sdrfile.get_files(stream=args.stream,post_pipeline_mode=post_pipeline_mode,dry_run=args.dry_run)
if args.all:
# do not hide any attribute
pass
else:
# hide non essential attributes
files=sdreducecol.run(files)
if not args.dry_run:
if len(files)>0:
files=sdcolumnfilter.run(files,args.column)
sdprint.print_format(files,args.format)
else:
print_stderr("File not found")
def file_dump(args):
import sdrfile, sddeferredafter, sdcolumnfilter, sdreducecol
sddeferredafter.add_default_parameter(args.stream, 'limit',
sdconfig.get_default_limit('dump'))
if args.raw_mode:
post_pipeline_mode = None
args.all = True # we force '--all' option when '--raw_mode' option is set
else:
post_pipeline_mode = 'file'
files = sdrfile.get_files(stream=args.stream,
post_pipeline_mode=post_pipeline_mode,
dry_run=args.dry_run)
if args.all:
# do not hide any attribute
pass
else:
# hide non essential attributes
files = sdreducecol.run(files)
if not args.dry_run:
if len(files) > 0:
files = sdcolumnfilter.run(files, args.column)
sdprint.print_format(files, args.format)
else:
print_stderr("File not found")
def complete_parameter(self):
self.parameter.extend(
sdsessionparam.get_serialized_session_facets()
) # TODO: maybe set session facets as default parameter being overwrited by parameter
sddeferredafter.add_default_parameter(
self.parameter, 'limit', sdsessionparam.get_value('limit'))
sddeferredafter.add_default_parameter(
self.parameter, 'verbose', sdsessionparam.get_value('verbose'))
def variable_list(args):
import sddeferredafter
sddeferredafter.add_default_parameter(
args.stream, 'limit', sdconfig.get_default_limit('list')
) # note: in variable mode, total number of row is given by: "total+=#variable for each ds"
print_stderr('Not implemented yet.')
"""
def file_list(args):
import sddeferredafter, sdlfile
sddeferredafter.add_default_parameter(args.stream,'limit',20)
files=sdlfile.get_files(stream=args.stream,dry_run=args.dry_run)
if len(files)==0:
print_stderr("File not found")
else:
sdlfile.print_list(files)
def file_list(args):
import sddeferredafter, sdlfile
sddeferredafter.add_default_parameter(args.stream, 'limit',
sdconfig.get_default_limit('list'))
files = sdlfile.get_files(stream=args.stream, dry_run=args.dry_run)
if len(files) == 0:
print_stderr("File not found")
else:
sdlfile.print_list(files)
def dataset_list(args):
import sddeferredafter
sddeferredafter.add_default_parameter(args.stream,'limit',20)
import sdldataset
datasets=sdldataset.get_datasets(stream=args.stream,dry_run=args.dry_run)
if len(datasets)==0:
print_stderr('Dataset not found')
else:
sdldataset.print_list(datasets)
def variable_search(args):
import sddeferredafter, sdrdataset, sdrvariable
sddeferredafter.add_default_parameter(args.stream,'limit',15) # note: in variable mode, total number of row is given by: "total+=#variable for each ds"
sddeferredafter.add_forced_parameter(args.stream,'fields',variable_light_fields)
datasets=sdrdataset.get_datasets(stream=args.stream,dry_run=args.dry_run)
if len(datasets)==0:
print "Variable not found"
else:
sdrvariable.print_list(datasets)
def dataset_list(args):
import sddeferredafter
sddeferredafter.add_default_parameter(args.stream, 'limit',
sdconfig.get_default_limit('list'))
import sdldataset
datasets = sdldataset.get_datasets(stream=args.stream,
dry_run=args.dry_run)
if len(datasets) == 0:
print_stderr('Dataset not found')
else:
sdldataset.print_list(datasets)
def dataset_dump(args):
import sdrdataset, sddeferredafter, sdcolumnfilter
sddeferredafter.add_default_parameter(args.stream,'limit',100)
post_pipeline_mode=None if args.raw_mode else 'dataset'
files=sdrdataset.get_datasets(stream=args.stream,post_pipeline_mode=post_pipeline_mode,dry_run=args.dry_run)
if not args.dry_run:
if len(files)>0:
files=sdcolumnfilter.run(files,args.column)
sdprint.print_format(files,args.format)
else:
print_stderr('Dataset not found')
def variable_search(args):
import sddeferredafter, sdrdataset, sdrvariable, sdfields, sdearlystreamutils
sddeferredafter.add_default_parameter(args.stream,'limit',args.limit) # TAGJ43JK3J43
lpcme=sdearlystreamutils.test_facet_value_early(args.stream,'local_path_format','custom') # lpcme means 'Local Path Custom Mode Enabled'
fields_=sdfields.get_all_variable_fields() if lpcme else sdfields.get_variable_light_fields()
sddeferredafter.add_forced_parameter(args.stream,'fields',fields_)
datasets=sdrdataset.get_datasets(stream=args.stream,dry_run=args.dry_run)
if len(datasets)==0:
print "Variable not found"
else:
sdrvariable.print_list(datasets,args.limit) # TAGJ43JK3J43
def file_pexec(args):
import sdrfile, sddeferredafter, sdpporder, sddb
sddeferredafter.add_default_parameter(args.stream,'limit',8000)
files=sdrfile.get_files(stream=args.stream)
if len(files)>0:
for f in files:
if f['status']==sdconst.TRANSFER_STATUS_DONE:
sdpporder.submit(args.order_name,sdconst.SA_TYPE_FILE,f['project'],f['model'],f['dataset_local_path'],variable=f['variable'],filename=f['filename'],commit=False)
sddb.conn.commit()
print_stderr("Post-processing task successfully submitted")
else:
print_stderr("File not found")
def dataset_dump(args):
import sdrdataset, sddeferredafter, sdcolumnfilter
sddeferredafter.add_default_parameter(args.stream, 'limit',
sdconfig.get_default_limit('dump'))
post_pipeline_mode = None if args.raw_mode else 'dataset'
files = sdrdataset.get_datasets(stream=args.stream,
post_pipeline_mode=post_pipeline_mode,
dry_run=args.dry_run)
if not args.dry_run:
if len(files) > 0:
files = sdcolumnfilter.run(files, args.column)
sdprint.print_format(files, args.format)
else:
print_stderr('Dataset not found')
def dataset_pexec(args):
import sdrdataset, sddeferredafter, sdstream, sdpporder, sddb
sddeferredafter.add_default_parameter(args.stream,'limit',8000) # add default limit
datasets=sdrdataset.get_datasets(stream=args.stream)
if len(datasets)>0:
for d in datasets:
if d['status']==sdconst.DATASET_STATUS_COMPLETE:
sdpporder.submit(args.order_name,sdconst.SA_TYPE_DATASET,d['project'],d['model'],d['local_path'],commit=False)
sddb.conn.commit()
print_stderr("Post-processing task successfully submitted")
else:
print_stderr('Dataset not found')
def dataset_search(args):
import sddeferredafter, sdrdataset, sdstream
sddeferredafter.add_default_parameter(args.stream,'limit',100) # add default limit
sddeferredafter.add_forced_parameter(args.stream,'fields',dataset_light_fields)
datasets=sdrdataset.get_datasets(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if len(datasets)==0:
print_stderr('Dataset not found')
else:
if args.replica:
sdrdataset.print_replica_list(datasets)
else:
sdrdataset.print_list(datasets)
def variable_pexec(args):
import sdrdataset, sdrvariable, sddeferredafter, sdpporder, sddb
sddeferredafter.add_default_parameter(args.stream,'limit', 8000) # note: in variable mode, total number of row is given by: "total+=#variable for each ds"
datasets=sdrdataset.get_datasets(stream=args.stream)
if len(datasets)>0:
for d in datasets:
if d['status']==sdconst.DATASET_STATUS_COMPLETE: # TODO: use VARIABLE_COMPLETE here !
for v in d['variable']:
sdpporder.submit(args.order_name,sdconst.SA_TYPE_AGGREGATION,d['project'],d['model'],d['local_path'],variable=v,commit=False)
sddb.conn.commit()
print_stderr("Post-processing task successfully submitted")
else:
print_stderr('Variable not found')
def esgf_search_api(args):
import sdrfile, sddeferredafter
# tuning: note that we don't reduce the number of field returned here.
# Maybe change that to optimise download time / reduce bandwidth footprint.
sddeferredafter.add_default_parameter(args.stream, 'limit', args.limit)
files = sdrfile.get_files(stream=args.stream, dry_run=args.dry_run)
if not args.dry_run:
if len(files) == 0:
print_stderr("File not found")
else:
if args.replica:
sdrfile.print_replica_list(files)
else:
sdrfile.print_list(files)
def file_search(args):
import sdrfile, sddeferredafter
# tuning: note that we don't reduce the number of field returned here.
# Maybe change that to optimise download time / reduce bandwidth footprint.
sddeferredafter.add_default_parameter(args.stream,'limit',100)
files=sdrfile.get_files(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if len(files)==0:
print_stderr("File not found")
else:
if args.replica:
sdrfile.print_replica_list(files)
else:
sdrfile.print_list(files)
def dataset_search(args):
import sddeferredafter, sdrdataset, sdstream, sdfields, sdearlystreamutils
sddeferredafter.add_default_parameter(args.stream,'limit',args.limit)
lpcme=sdearlystreamutils.test_facet_value_early(args.stream,'local_path_format','custom') # lpcme means 'Local Path Custom Mode Enabled'
fields_=sdfields.get_all_dataset_fields() if lpcme else sdfields.get_dataset_light_fields()
sddeferredafter.add_forced_parameter(args.stream,'fields',fields_)
datasets=sdrdataset.get_datasets(stream=args.stream,dry_run=args.dry_run)
if not args.dry_run:
if len(datasets)==0:
print_stderr('Dataset not found')
else:
if args.replica:
sdrdataset.print_replica_list(datasets)
else:
sdrdataset.print_list(datasets)
def variable_search(args):
import sddeferredafter, sdrdataset, sdrvariable, sdfields, sdearlystreamutils
sddeferredafter.add_default_parameter(args.stream, 'limit',
args.limit) # TAGJ43JK3J43
lpcme = sdearlystreamutils.test_facet_value_early(
args.stream, 'local_path_format',
'custom') # lpcme means 'Local Path Custom Mode Enabled'
fields_ = sdfields.get_all_variable_fields(
) if lpcme else sdfields.get_variable_light_fields()
sddeferredafter.add_forced_parameter(args.stream, 'fields', fields_)
datasets = sdrdataset.get_datasets(stream=args.stream,
dry_run=args.dry_run)
if len(datasets) == 0:
print "Variable not found"
else:
sdrvariable.print_list(datasets, args.limit) # TAGJ43JK3J43
def dataset_search(args):
import sddeferredafter, sdrdataset, sdstream, sdfields, sdearlystreamutils
sddeferredafter.add_default_parameter(args.stream, 'limit', args.limit)
lpcme = sdearlystreamutils.test_facet_value_early(
args.stream, 'local_path_format',
'custom') # lpcme means 'Local Path Custom Mode Enabled'
fields_ = sdfields.get_all_dataset_fields(
) if lpcme else sdfields.get_dataset_light_fields()
sddeferredafter.add_forced_parameter(args.stream, 'fields', fields_)
datasets = sdrdataset.get_datasets(stream=args.stream,
dry_run=args.dry_run)
if not args.dry_run:
if len(datasets) == 0:
print_stderr('Dataset not found')
else:
if args.replica:
sdrdataset.print_replica_list(datasets)
else:
sdrdataset.print_list(datasets)
def get(args):
import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils
# hack
# see TAG43534FSFS
if args.quiet:
args.verbosity = 0
if args.verify_checksum and args.network_bandwidth_test:
print_stderr(
"'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set."
)
return 1
stream = syndautils.get_stream(subcommand=args.subcommand,
parameter=args.parameter,
selection_file=args.selection_file)
if args.openid and args.password:
# use credential from CLI
oid = args.openid
pwd = args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid = sdconfig.openid
pwd = sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).' %
sdconfig.credential_file)
return 1
# retrieve certificate
sdlogon.renew_certificate(oid, pwd, force_renew_certificate=False)
http_client = sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET
# local_path
#
# 'synda get' subcommand currently force local_path to the following construct:
# '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may
# change in the future.
#
if args.dest_folder is None:
local_path_prefix = os.getcwd() # current working directory
else:
local_path_prefix = args.dest_folder
# BEWARE
#
# when set in CLI parameter, url is usually an ESGF facet, and as so should
# be sent to the search-api as other facets
# BUT
# we want a special behaviour here (i.e. with 'synda get' command) with url:
# if url is set by user, we DON'T call search-api operator. Instead, we
# download the url directly.
urls = sdearlystreamutils.get_facet_values_early(stream, 'url')
if len(urls) == 0:
# no url in stream: switch to search-api operator mode
sddeferredafter.add_default_parameter(stream, 'limit', 5)
sddeferredafter.add_forced_parameter(stream, 'local_path_format',
'notree')
files = sdrfile.get_files(
stream=stream, post_pipeline_mode='file', dry_run=args.dry_run
) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent
if not args.dry_run:
if len(files) > 0:
# compute metric
total_size = sum(int(f['size']) for f in files)
total_size = humanize.naturalsize(total_size, gnu=False)
print_stderr(
'%i file(s) will be downloaded for a total size of %s.' %
(len(files), total_size))
status = sddirectdownload.run(
files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum,
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status != 0:
return 1
else:
print_stderr("File not found")
return 1
else:
for f in files:
size = humanize.naturalsize(f['size'], gnu=False)
print '%-12s %s' % (size, f['filename'])
elif len(urls) > 0:
# url(s) found in stream: search-api operator not needed (download url directly)
# TAGDSFDF432F
if args.verify_checksum:
print_stderr(
"To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url."
)
return 1
# TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url))
files = []
for url in urls:
filename = os.path.basename(url)
local_path = filename
f = dict(local_path=local_path, url=url)
files.append(f)
status = sddirectdownload.run(
files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status != 0:
return 1
else:
assert False
return 0
def get(args):
import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils
# hack
# see TAG43534FSFS
if args.quiet:
args.verbosity=0
if args.verify_checksum and args.network_bandwidth_test:
print_stderr("'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set.")
return 1
stream=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file)
if args.openid and args.password:
# use credential from CLI
oid=args.openid
pwd=args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid=sdconfig.openid
pwd=sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).'%sdconfig.credential_file)
return 1
# retrieve certificate
sdlogon.renew_certificate(oid,pwd,force_renew_certificate=False)
http_client=sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET
# local_path
#
# 'synda get' subcommand currently force local_path to the following construct:
# '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may
# change in the future.
#
if args.dest_folder is None:
local_path_prefix=os.getcwd() # current working directory
else:
local_path_prefix=args.dest_folder
# BEWARE
#
# when set in CLI parameter, url is usually an ESGF facet, and as so should
# be sent to the search-api as other facets
# BUT
# we want a special behaviour here (i.e. with 'synda get' command) with url:
# if url is set by user, we DON'T call search-api operator. Instead, we
# download the url directly.
urls=sdearlystreamutils.get_facet_values_early(stream,'url')
if len(urls)==0:
# no url in stream: switch to search-api operator mode
sddeferredafter.add_default_parameter(stream,'limit',5)
sddeferredafter.add_forced_parameter(stream,'local_path_format','notree')
files=sdrfile.get_files(stream=stream,post_pipeline_mode='file',dry_run=args.dry_run) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent
if not args.dry_run:
if len(files)>0:
# compute metric
total_size=sum(int(f['size']) for f in files)
total_size=humanize.naturalsize(total_size,gnu=False)
print_stderr('%i file(s) will be downloaded for a total size of %s.'%(len(files),total_size))
status=sddirectdownload.run(files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum,
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status!=0:
return 1
else:
print_stderr("File not found")
return 1
else:
for f in files:
size=humanize.naturalsize(f['size'],gnu=False)
print '%-12s %s'%(size,f['filename'])
elif len(urls)>0:
# url(s) found in stream: search-api operator not needed (download url directly)
# TAGDSFDF432F
if args.verify_checksum:
print_stderr("To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url.")
return 1
# TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url))
files=[]
for url in urls:
filename=os.path.basename(url)
local_path=filename
f=dict(local_path=local_path,url=url)
files.append(f)
status=sddirectdownload.run(files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status!=0:
return 1
else:
assert False
return 0
def complete_parameter(self):
self.parameter.extend(sdsessionparam.get_serialized_session_facets()) # TODO: maybe set session facets as default parameter being overwrited by parameter
sddeferredafter.add_default_parameter(self.parameter,'limit',sdsessionparam.get_value('limit'))
sddeferredafter.add_default_parameter(self.parameter,'verbose',sdsessionparam.get_value('verbose'))
