def get_pid_from_filename(f):
match=re.search('^.*sdt_transient_storage_([^_]+)_[^_]+$',f)
if match!=None:
pid=match.group(1)
else:
raise sdexception.SDException("SYNDTDBF-001","Incorrect filename format (%s)"%(f,))
return pid
def extract_project(path):
m = re.search('^([^/]+)/', path)
if m != None:
project = m.group(1)
return project
else:
raise sdexception.SDException('SDLOCALP-006',
'Incorrect value (path=%s)' % path)
def file_search(args):
import sdfilesearch
if sdconfig.metadata_server_type not in sdconst.METADATA_SERVER_TYPES:
raise sdexception.SDException(
'SDTSACTI-001', 'Incorrect metadata server type (%s)' %
sdconfig.metadata_server_type)
return getattr(sdfilesearch, sdconfig.metadata_server_type)(args)
def parse_rule(name, body):
match = re.search('^s\|([^|]+)\|([^|]*)\|$', body)
if match != None:
from_string = match.group(1)
to_string = match.group(2)
else:
raise sdexception.SDException(
"SYNDTRAN-001",
"Incorrect format for '%s' parameter (%s)" % (name, body))
return (from_string, to_string)
def get_exclude(args):
li=[]
if args.exclude_from is None:
return []
if not os.path.isfile(args.exclude_from):
raise sdexception.SDException('SDUPGRAD-108','exclude-from file not found (%s)'%args.exclude_from)
with open(args.exclude_from) as f:
li = [line.rstrip('\r\n') for line in f]
return li
def main_pipeline(files, mode=None):
assert isinstance(files, list)
if mode == 'file':
files = sdgenericpipeline.run(files)
files = sdfilepipeline.run(files=files)
elif mode == 'dataset':
files = sdgenericpipeline.run(files)
files = sddatasetpipeline.run(files=files)
elif mode == 'generic':
files = sdgenericpipeline.run(files)
else:
raise sdexception.SDException("SDPIPELI-001",
"Incorrect mode (%s)" % mode)
return files
def transform_local_path_product(files, key):
for f in files:
fmt = sdpostpipelineutils.get_attached_parameter(
f, 'local_path_product_format',
sdconst.DEFAULT_LOCAL_PATH_PRODUCT_FORMAT)
if fmt == "normal":
pass
elif fmt == "merge":
f[key] = sdproduct.replace_output12_product_with_output_product(
f[key])
elif fmt == "remove":
f[key] = sdproduct.remove_product(f[key])
else:
raise sdexception.SDException(
'SDLOCALP-002',
"'local_path_product_format' is incorrect (%s)" % fmt)
return files
def set_stream_type(args):
import sddeferredbefore
# Set the sdtream type (aka search-API 'type').
#
# Note that arg.type_ is NOT the same thing as the stream type (aka
# search-API type). arg.type_ is only used locally to format the
# listing presented to user, while the stream type is the one sent
# to the ESGF service to retrieve data. For example,
# SA_TYPE_AGGREGATION is used by arg.type_ to make some change in
# the output, but search-API don't know about this type (i.e. for
# most project, you can't list anything by using this type). Also
# most modules of Synda behave the same way as search-API: they
# don't know about SA_TYPE_AGGREGATION. SA_TYPE_AGGREGATION is ONLY
# used in Synda upstream code to make some local display
# modifications.
#
# So what we do here is choose which is the search-API type we need
# (dataset, file) for the listing type asked by user (i.e.
# variable, dataset, file)
#
# But note that in most case, search-API 'type' will be overrided
# later anyway as it is forced in dedicated modules (e.g. in
# sdrdataset, sdrfile, etc..).
#
# Also note that we 'force' (i.e. not 'default') the parameter here, so to
# prevent user to set it. We do this because if user use '-f' option and
# type=Dataset, the display type will not fit the type of data fetched from
# search-API).
#
if args.type_ in (sdconst.SA_TYPE_AGGREGATION, sdconst.SA_TYPE_DATASET):
sddeferredbefore.add_forced_parameter(args.stream, 'type',
sdconst.SA_TYPE_DATASET)
elif args.type_ in (sdconst.SA_TYPE_FILE, ):
sddeferredbefore.add_forced_parameter(args.stream, 'type',
sdconst.SA_TYPE_FILE)
else:
raise sdexception.SDException('SDASYNDA-001',
'Unknown type (%s)' % args.type_)
def run(i__queries):
"""This method contains the retry mecanism."""
# check
for q in i__queries:
if sdconst.IDXHOSTMARK not in q['url']:
raise sdexception.SDException('SDPROXMT-044','Incorrect query: host must not be set at this step')
# retry loop
max_retry=6
i=0
metadata=sdtypes.Metadata()
l__queries=i__queries
while i < max_retry:
(success,errors)=run_helper(l__queries)
metadata.slurp(success) # warning: success is modified here
if len(errors)>0:
sdlog.info("SDPROXMT-082","%d search-API queries failed"%(len(errors),))
sdlog.info("SDPROXMT-083","retry 'failed search-API queries'")
l__queries=errors
i+=1
continue
else:
if i>0:
sdlog.info("SDPROXMT-089","retry succeeded")
break
if len(errors)>0:
sdlog.error("SDPROXMT-084","max retry iteration reached. %d queries did not succeed"%(len(errors),))
return metadata