def dataset_show(args): import sdearlystreamutils # check li=sdearlystreamutils.get_facet_values_early(args.stream,'instance_id') if len(li)==0: print_stderr('Please specify a dataset name.') return 1 elif len(li)>1: print_stderr('Too many arguments.') return 1 if args.localsearch: import sdldataset dataset=sdldataset.get_dataset(stream=args.stream,dry_run=args.dry_run) if not args.dry_run: if dataset is None: print_stderr("Dataset not found") else: sdldataset.print_details(dataset) else: import sdrdataset dataset=sdrdataset.get_dataset(stream=args.stream,dry_run=args.dry_run) if not args.dry_run: if dataset is None: print_stderr("Dataset not found") else: sdrdataset.print_details(dataset,verbose=args.verbose)
def dataset_version(args): import sdremoteparam, syndautils, sdearlystreamutils # don't be misled about identifiers here: sdinference produces search-api # key (always do) name i.e. instance_id, and I use Synda style variable name # i.e. dataset_functional_id (for better readability). li = sdearlystreamutils.get_facet_values_early(args.stream, 'instance_id') if len(li) == 0: print_stderr('Please specify a dataset name.') return 1 elif len(li) > 1: print_stderr('Too many arguments.') return 1 else: dataset_functional_id = li[0] dataset_functional_id_without_version = syndautils.strip_dataset_version( dataset_functional_id) params = sdremoteparam.run(pname='version', facets_group={ 'type': [sdconst.SA_TYPE_DATASET], 'master_id': [dataset_functional_id_without_version] }, dry_run=args.dry_run) # TODO: func for code below items = params.get('version', []) for item in items: print item.name
def dataset_show(args): import sdearlystreamutils # check li = sdearlystreamutils.get_facet_values_early(args.stream, 'instance_id') if len(li) == 0: print_stderr('Please specify a dataset name.') return 1 elif len(li) > 1: print_stderr('Too many arguments.') return 1 if args.localsearch: import sdldataset dataset = sdldataset.get_dataset(stream=args.stream, dry_run=args.dry_run) if not args.dry_run: if dataset is None: print_stderr("Dataset not found") else: sdldataset.print_details(dataset) else: import sdrdataset dataset = sdrdataset.get_dataset(stream=args.stream, dry_run=args.dry_run) if not args.dry_run: if dataset is None: print_stderr("Dataset not found") else: sdrdataset.print_details(dataset, verbose=args.verbose)
def get_projects(selection): """This method returns project (standalone pending, standalone non-pending or as part of an identifier) from CLI parameter or selection file.""" project = [] # project with keyname if 'project' in selection.facets: project += selection.facets['project'] # project without keyname # WARNING # # The code below uses sdinference and query the database to retrieve ESGF parameters. # Doing those things here may raise circular dependencies # as well as making the whole thing very complex. # # We do this to make this syntax work (i.e. project value without key) # synda search GeoMIP # # Note that this syntax always works (i.e. load the project level default file), even without this code. # synda search project=GeoMIP # # pending_projects = sdearlystreamutils.get_facet_values_early( [selection.facets], 'project', extract_item=True ) # project without keyname or project as part of an identifier. li = pending_projects + project li = list(set(li)) # remove duplicate return li
def get_projects(selection): """This method returns project (standalone pending, standalone non-pending or as part of an identifier) from CLI parameter or selection file.""" project=[] # project with keyname if 'project' in selection.facets: project+=selection.facets['project'] # project without keyname # WARNING # # The code below uses sdinference and query the database to retrieve ESGF parameters. # Doing those things here may raise circular dependencies # as well as making the whole thing very complex. # # We do this to make this syntax work (i.e. project value without key) # synda search GeoMIP # # Note that this syntax always works (i.e. load the project level default file), even without this code. # synda search project=GeoMIP # # pending_projects=sdearlystreamutils.get_facet_values_early([selection.facets],'project',extract_item=True) # project without keyname or project as part of an identifier. li=pending_projects+project li=list(set(li)) # remove duplicate return li
def file_show(args): import sdearlystreamutils # check li = sdearlystreamutils.get_facet_values_early( args.stream, 'instance_id') # check if 'instance_id' exists if len(li) == 0: # 'instance_id' is not found on cli li = sdearlystreamutils.get_facet_values_early( args.stream, 'title') # check if 'title' exists if len(li) == 0: # 'title' is not found on cli # no identifier found, we stop the processing print_stderr('Please specify a file identifier (id or filename).') return 1 elif len(li) > 1: print_stderr('Too many arguments.') return 1 elif len(li) > 1: print_stderr('Too many arguments.') return 1 # main if args.localsearch: import sdlfile file = sdlfile.get_file(stream=args.stream, dry_run=args.dry_run) if not args.dry_run: if file is None: print_stderr("File not found") else: sdlfile.print_details(file) else: import sdrfile file = sdrfile.get_file(stream=args.stream, dry_run=args.dry_run) if not args.dry_run: if file is None: print_stderr("File not found") else: sdrfile.print_details(file)
def file_show(args): import sdearlystreamutils # check li=sdearlystreamutils.get_facet_values_early(args.stream,'instance_id') # check if 'instance_id' exists if len(li)==0: # 'instance_id' is not found on cli li=sdearlystreamutils.get_facet_values_early(args.stream,'title') # check if 'title' exists if len(li)==0: # 'title' is not found on cli # no identifier found, we stop the processing print_stderr('Please specify a file identifier (id or filename).') return 1 elif len(li)>1: print_stderr('Too many arguments.') return 1 elif len(li)>1: print_stderr('Too many arguments.') return 1 # main if args.localsearch: import sdlfile file=sdlfile.get_file(stream=args.stream,dry_run=args.dry_run) if not args.dry_run: if file is None: print_stderr("File not found") else: sdlfile.print_details(file) else: import sdrfile file=sdrfile.get_file(stream=args.stream,dry_run=args.dry_run) if not args.dry_run: if file is None: print_stderr("File not found") else: sdrfile.print_details(file)
def apache_default_listing(args): import sdearlystreamutils, sdhtmlbasic urls = sdearlystreamutils.get_facet_values_early(args.stream, 'url') if len(urls) == 0: # no url in stream print_stderr("Incorrect argument: please specify an url") return 1 else: files = sdhtmlbasic.get_files(stream=args.stream, dry_run=args.dry_run) if not args.dry_run: if len(files) == 0: print_stderr("File not found") else: sdhtmlbasic.print_list(files)
def dataset_version(args): import sdremoteparam,syndautils,sdearlystreamutils # don't be misled about identifiers here: sdinference produces search-api # key (always do) name i.e. instance_id, and I use Synda style variable name # i.e. dataset_functional_id (for better readability). li=sdearlystreamutils.get_facet_values_early(args.stream,'instance_id') if len(li)==0: print_stderr('Please specify a dataset name.') return 1 elif len(li)>1: print_stderr('Too many arguments.') return 1 else: dataset_functional_id=li[0] dataset_functional_id_without_version=syndautils.strip_dataset_version(dataset_functional_id) params=sdremoteparam.run(pname='version',facets_group={'type':[sdconst.SA_TYPE_DATASET],'master_id':[dataset_functional_id_without_version]},dry_run=args.dry_run) # TODO: func for code below items=params.get('version',[]) for item in items: print item.name
def open_(args): import sdview,syndautils,sdsandbox,sdtypes,sdconst,sdearlystreamutils stream=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file) # check li=sdearlystreamutils.get_facet_values_early(stream,'instance_id') # check if 'instance_id' exists if len(li)==0: # 'instance_id' is not found on cli li=sdearlystreamutils.get_facet_values_early(stream,'title') # check if 'title' exists if len(li)==0: # 'title' is not found on cli # no identifier found, we stop the processing print_stderr('Please specify a file identifier (id or filename).') return 1 elif len(li)>1: print_stderr('Too many arguments.') return 1 elif len(li)>1: print_stderr('Too many arguments.') return 1 # discovery import sdlfile file_=sdlfile.get_file(stream=stream) if file_ is None: import sdrfile file_=sdrfile.get_file(stream=stream) if file_ is None: print_stderr("File not found") return 2 # cast f=sdtypes.File(**file_) # check if file exists locally if f.status==sdconst.TRANSFER_STATUS_DONE: file_local_path=f.get_full_local_path() elif sdsandbox.file_exists(f.filename): file_local_path=sdsandbox.get_file_path(f.filename) else: file_local_path=None # download (if not done already) if file_local_path is None: status=sddirectdownload.run([file_], verbosity=1) if status!=0: return 1 # open file in external viewer sdview.open_(file_local_path,f.variable,args.geometry) return 0
def get(args): import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils # hack # see TAG43534FSFS if args.quiet: args.verbosity=0 if args.verify_checksum and args.network_bandwidth_test: print_stderr("'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set.") return 1 stream=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file) if args.openid and args.password: # use credential from CLI oid=args.openid pwd=args.password else: # use credential from file if sdconfig.is_openid_set(): oid=sdconfig.openid pwd=sdconfig.password else: print_stderr('Error: OpenID not set in configuration file (%s).'%sdconfig.credential_file) return 1 # retrieve certificate sdlogon.renew_certificate(oid,pwd,force_renew_certificate=False) http_client=sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET # local_path # # 'synda get' subcommand currently force local_path to the following construct: # '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may # change in the future. # if args.dest_folder is None: local_path_prefix=os.getcwd() # current working directory else: local_path_prefix=args.dest_folder # BEWARE # # when set in CLI parameter, url is usually an ESGF facet, and as so should # be sent to the search-api as other facets # BUT # we want a special behaviour here (i.e. with 'synda get' command) with url: # if url is set by user, we DON'T call search-api operator. Instead, we # download the url directly. urls=sdearlystreamutils.get_facet_values_early(stream,'url') if len(urls)==0: # no url in stream: switch to search-api operator mode sddeferredafter.add_default_parameter(stream,'limit',5) sddeferredafter.add_forced_parameter(stream,'local_path_format','notree') files=sdrfile.get_files(stream=stream,post_pipeline_mode='file',dry_run=args.dry_run) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent if not args.dry_run: if len(files)>0: # compute metric total_size=sum(int(f['size']) for f in files) total_size=humanize.naturalsize(total_size,gnu=False) print_stderr('%i file(s) will be downloaded for a total size of %s.'%(len(files),total_size)) status=sddirectdownload.run(files, args.timeout, args.force, http_client, local_path_prefix, verify_checksum=args.verify_checksum, network_bandwidth_test=args.network_bandwidth_test, debug=True, verbosity=args.verbosity, buffered=False, hpss=args.hpss) if status!=0: return 1 else: print_stderr("File not found") return 1 else: for f in files: size=humanize.naturalsize(f['size'],gnu=False) print '%-12s %s'%(size,f['filename']) elif len(urls)>0: # url(s) found in stream: search-api operator not needed (download url directly) # TAGDSFDF432F if args.verify_checksum: print_stderr("To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url.") return 1 # TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url)) files=[] for url in urls: filename=os.path.basename(url) local_path=filename f=dict(local_path=local_path,url=url) files.append(f) status=sddirectdownload.run(files, args.timeout, args.force, http_client, local_path_prefix, verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F network_bandwidth_test=args.network_bandwidth_test, debug=True, verbosity=args.verbosity, buffered=False, hpss=args.hpss) if status!=0: return 1 else: assert False return 0
def open_(args): import sdview, syndautils, sdsandbox, sdtypes, sdconst, sdearlystreamutils stream = syndautils.get_stream(subcommand=args.subcommand, parameter=args.parameter, selection_file=args.selection_file) # check li = sdearlystreamutils.get_facet_values_early( stream, 'instance_id') # check if 'instance_id' exists if len(li) == 0: # 'instance_id' is not found on cli li = sdearlystreamutils.get_facet_values_early( stream, 'title') # check if 'title' exists if len(li) == 0: # 'title' is not found on cli # no identifier found, we stop the processing print_stderr('Please specify a file identifier (id or filename).') return 1 elif len(li) > 1: print_stderr('Too many arguments.') return 1 elif len(li) > 1: print_stderr('Too many arguments.') return 1 # discovery import sdlfile file_ = sdlfile.get_file(stream=stream) if file_ is None: import sdrfile file_ = sdrfile.get_file(stream=stream) if file_ is None: print_stderr("File not found") return 2 # cast f = sdtypes.File(**file_) # check if file exists locally if f.status == sdconst.TRANSFER_STATUS_DONE: file_local_path = f.get_full_local_path() elif sdsandbox.file_exists(f.filename): file_local_path = sdsandbox.get_file_path(f.filename) else: file_local_path = None # download (if not done already) if file_local_path is None: status = sddirectdownload.run([file_], verbosity=1) if status != 0: return 1 # open file in external viewer sdview.open_(file_local_path, f.variable, args.geometry) return 0
def get(args): import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils # hack # see TAG43534FSFS if args.quiet: args.verbosity = 0 if args.verify_checksum and args.network_bandwidth_test: print_stderr( "'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set." ) return 1 stream = syndautils.get_stream(subcommand=args.subcommand, parameter=args.parameter, selection_file=args.selection_file) if args.openid and args.password: # use credential from CLI oid = args.openid pwd = args.password else: # use credential from file if sdconfig.is_openid_set(): oid = sdconfig.openid pwd = sdconfig.password else: print_stderr('Error: OpenID not set in configuration file (%s).' % sdconfig.credential_file) return 1 # retrieve certificate sdlogon.renew_certificate(oid, pwd, force_renew_certificate=False) http_client = sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET # local_path # # 'synda get' subcommand currently force local_path to the following construct: # '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may # change in the future. # if args.dest_folder is None: local_path_prefix = os.getcwd() # current working directory else: local_path_prefix = args.dest_folder # BEWARE # # when set in CLI parameter, url is usually an ESGF facet, and as so should # be sent to the search-api as other facets # BUT # we want a special behaviour here (i.e. with 'synda get' command) with url: # if url is set by user, we DON'T call search-api operator. Instead, we # download the url directly. urls = sdearlystreamutils.get_facet_values_early(stream, 'url') if len(urls) == 0: # no url in stream: switch to search-api operator mode sddeferredafter.add_default_parameter(stream, 'limit', 5) sddeferredafter.add_forced_parameter(stream, 'local_path_format', 'notree') files = sdrfile.get_files( stream=stream, post_pipeline_mode='file', dry_run=args.dry_run ) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent if not args.dry_run: if len(files) > 0: # compute metric total_size = sum(int(f['size']) for f in files) total_size = humanize.naturalsize(total_size, gnu=False) print_stderr( '%i file(s) will be downloaded for a total size of %s.' % (len(files), total_size)) status = sddirectdownload.run( files, args.timeout, args.force, http_client, local_path_prefix, verify_checksum=args.verify_checksum, network_bandwidth_test=args.network_bandwidth_test, debug=True, verbosity=args.verbosity, buffered=False, hpss=args.hpss) if status != 0: return 1 else: print_stderr("File not found") return 1 else: for f in files: size = humanize.naturalsize(f['size'], gnu=False) print '%-12s %s' % (size, f['filename']) elif len(urls) > 0: # url(s) found in stream: search-api operator not needed (download url directly) # TAGDSFDF432F if args.verify_checksum: print_stderr( "To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url." ) return 1 # TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url)) files = [] for url in urls: filename = os.path.basename(url) local_path = filename f = dict(local_path=local_path, url=url) files.append(f) status = sddirectdownload.run( files, args.timeout, args.force, http_client, local_path_prefix, verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F network_bandwidth_test=args.network_bandwidth_test, debug=True, verbosity=args.verbosity, buffered=False, hpss=args.hpss) if status != 0: return 1 else: assert False return 0