def get_pid_from_filename(f): match=re.search('^.*sdt_transient_storage_([^_]+)_[^_]+$',f) if match!=None: pid=match.group(1) else: raise sdexception.SDException("SYNDTDBF-001","Incorrect filename format (%s)"%(f,)) return pid
def extract_project(path): m = re.search('^([^/]+)/', path) if m != None: project = m.group(1) return project else: raise sdexception.SDException('SDLOCALP-006', 'Incorrect value (path=%s)' % path)
def file_search(args): import sdfilesearch if sdconfig.metadata_server_type not in sdconst.METADATA_SERVER_TYPES: raise sdexception.SDException( 'SDTSACTI-001', 'Incorrect metadata server type (%s)' % sdconfig.metadata_server_type) return getattr(sdfilesearch, sdconfig.metadata_server_type)(args)
def parse_rule(name, body): match = re.search('^s\|([^|]+)\|([^|]*)\|$', body) if match != None: from_string = match.group(1) to_string = match.group(2) else: raise sdexception.SDException( "SYNDTRAN-001", "Incorrect format for '%s' parameter (%s)" % (name, body)) return (from_string, to_string)
def get_exclude(args): li=[] if args.exclude_from is None: return [] if not os.path.isfile(args.exclude_from): raise sdexception.SDException('SDUPGRAD-108','exclude-from file not found (%s)'%args.exclude_from) with open(args.exclude_from) as f: li = [line.rstrip('\r\n') for line in f] return li
def main_pipeline(files, mode=None): assert isinstance(files, list) if mode == 'file': files = sdgenericpipeline.run(files) files = sdfilepipeline.run(files=files) elif mode == 'dataset': files = sdgenericpipeline.run(files) files = sddatasetpipeline.run(files=files) elif mode == 'generic': files = sdgenericpipeline.run(files) else: raise sdexception.SDException("SDPIPELI-001", "Incorrect mode (%s)" % mode) return files
def transform_local_path_product(files, key): for f in files: fmt = sdpostpipelineutils.get_attached_parameter( f, 'local_path_product_format', sdconst.DEFAULT_LOCAL_PATH_PRODUCT_FORMAT) if fmt == "normal": pass elif fmt == "merge": f[key] = sdproduct.replace_output12_product_with_output_product( f[key]) elif fmt == "remove": f[key] = sdproduct.remove_product(f[key]) else: raise sdexception.SDException( 'SDLOCALP-002', "'local_path_product_format' is incorrect (%s)" % fmt) return files
def set_stream_type(args): import sddeferredbefore # Set the sdtream type (aka search-API 'type'). # # Note that arg.type_ is NOT the same thing as the stream type (aka # search-API type). arg.type_ is only used locally to format the # listing presented to user, while the stream type is the one sent # to the ESGF service to retrieve data. For example, # SA_TYPE_AGGREGATION is used by arg.type_ to make some change in # the output, but search-API don't know about this type (i.e. for # most project, you can't list anything by using this type). Also # most modules of Synda behave the same way as search-API: they # don't know about SA_TYPE_AGGREGATION. SA_TYPE_AGGREGATION is ONLY # used in Synda upstream code to make some local display # modifications. # # So what we do here is choose which is the search-API type we need # (dataset, file) for the listing type asked by user (i.e. # variable, dataset, file) # # But note that in most case, search-API 'type' will be overrided # later anyway as it is forced in dedicated modules (e.g. in # sdrdataset, sdrfile, etc..). # # Also note that we 'force' (i.e. not 'default') the parameter here, so to # prevent user to set it. We do this because if user use '-f' option and # type=Dataset, the display type will not fit the type of data fetched from # search-API). # if args.type_ in (sdconst.SA_TYPE_AGGREGATION, sdconst.SA_TYPE_DATASET): sddeferredbefore.add_forced_parameter(args.stream, 'type', sdconst.SA_TYPE_DATASET) elif args.type_ in (sdconst.SA_TYPE_FILE, ): sddeferredbefore.add_forced_parameter(args.stream, 'type', sdconst.SA_TYPE_FILE) else: raise sdexception.SDException('SDASYNDA-001', 'Unknown type (%s)' % args.type_)
def run(i__queries): """This method contains the retry mecanism.""" # check for q in i__queries: if sdconst.IDXHOSTMARK not in q['url']: raise sdexception.SDException('SDPROXMT-044','Incorrect query: host must not be set at this step') # retry loop max_retry=6 i=0 metadata=sdtypes.Metadata() l__queries=i__queries while i < max_retry: (success,errors)=run_helper(l__queries) metadata.slurp(success) # warning: success is modified here if len(errors)>0: sdlog.info("SDPROXMT-082","%d search-API queries failed"%(len(errors),)) sdlog.info("SDPROXMT-083","retry 'failed search-API queries'") l__queries=errors i+=1 continue else: if i>0: sdlog.info("SDPROXMT-089","retry succeeded") break if len(errors)>0: sdlog.error("SDPROXMT-084","max retry iteration reached. %d queries did not succeed"%(len(errors),)) return metadata