def download(url,full_local_path,checksum_type='md5',debug=False):
killed=False
script_stderr=None
transfer_protocol=sdutils.get_transfer_protocol(url)
if transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP:
if sdconfig.http_client==sdconst.HTTP_CLIENT_URLLIB:
(status,local_checksum)=sdget_urllib.download_file(url,full_local_path,checksum_type)
else:
(status,local_checksum,killed,script_stderr)=run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug)
elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP:
(status,local_checksum,killed,script_stderr)=run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug)
elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GLOBUS:
pass
else:
assert False
return (status,local_checksum,killed,script_stderr)
def replica_next(file_functional_id, args):
import sdrfile, sdmodify, sdfiledao, sdutils, sdconst
parameter = [
'keep_replica=true', 'nearest=false',
'file_functional_id=%s' % file_functional_id
]
files = sdrfile.get_files(parameter=parameter, dry_run=args.dry_run)
if not args.dry_run:
if len(files) == 0:
print_stderr("File not found in ESGF (file_functional_id=%s)" %
file_functional_id)
else:
replicas = [(f['url'], f['data_node']) for f in files]
file_ = sdfiledao.get_file(file_functional_id)
if file_ is not None:
if sdutils.get_transfer_protocol(
file_.url) == sdconst.TRANSFER_PROTOCOL_HTTP:
sdmodify.replica_next(file_, replicas)
else:
print_stderr(
"Incorrect protocol"
) # only http protocol is supported in 'synda replica' for now
else:
print_stderr("Local file not found")
def download(url,
full_local_path,
debug=False,
http_client=sdconfig.http_client,
timeout=sdconst.ASYNC_DOWNLOAD_HTTP_TIMEOUT,
verbosity=0,
buffered=True,
hpss=False):
killed = False
script_stderr = None
transfer_protocol = sdutils.get_transfer_protocol(url)
if transfer_protocol == sdconst.TRANSFER_PROTOCOL_HTTP:
if http_client == sdconst.HTTP_CLIENT_URLLIB:
status = sdget_urllib.download_file(url, full_local_path, timeout)
elif http_client == sdconst.HTTP_CLIENT_WGET:
li = prepare_args(url, full_local_path,
sdconfig.data_download_script_http, debug,
timeout, verbosity, hpss)
(status, script_stderr) = run_download_script(li, buffered)
killed = is_killed(transfer_protocol, status)
else:
assert False
elif transfer_protocol == sdconst.TRANSFER_PROTOCOL_GRIDFTP:
gridftp_opt = sdconfig.config.get('download', 'gridftp_opt')
if len(gridftp_opt) > 0:
os.environ["GRIDFTP_OPT"] = gridftp_opt
li = prepare_args(url, full_local_path,
sdconfig.data_download_script_gridftp, debug,
timeout, verbosity, hpss)
(status, script_stderr) = run_download_script(li, buffered)
killed = is_killed(transfer_protocol, status)
elif transfer_protocol == sdconst.TRANSFER_PROTOCOL_GLOBUS:
pass
else:
assert False
return (status, killed, script_stderr)
def download(url,full_local_path,checksum_type='md5',debug_level=0):
killed=False
script_stdxxx=None
transfer_protocol=sdutils.get_transfer_protocol(url)
if (transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP
and sdconfig.http_client==sdconst.HTTP_CLIENT_URLLIB):
(status,local_checksum)=sdnetutils.download_file(url,full_local_path,checksum_type)
else:
(status,local_checksum,killed,script_stdxxx)=run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug_level)
return (status,local_checksum,killed,script_stdxxx)
def download(url,full_local_path,debug=False,http_client=sdconfig.http_client,timeout=sdconst.ASYNC_DOWNLOAD_HTTP_TIMEOUT,verbosity=0,buffered=True,hpss=False):
killed=False
script_stderr=None
transfer_protocol=sdutils.get_transfer_protocol(url)
if transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP:
if http_client==sdconst.HTTP_CLIENT_URLLIB:
status=sdget_urllib.download_file(url,full_local_path,timeout)
elif http_client==sdconst.HTTP_CLIENT_WGET:
li=prepare_args(url,full_local_path,sdconfig.data_download_script_http,debug,timeout,verbosity,hpss)
(status,script_stderr)=run_download_script(li,buffered)
killed=is_killed(transfer_protocol,status)
else:
assert False
elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP:
gridftp_opt=sdconfig.config.get('download','gridftp_opt')
if len(gridftp_opt)>0:
os.environ["GRIDFTP_OPT"]=gridftp_opt
li=prepare_args(url,full_local_path,sdconfig.data_download_script_gridftp,debug,timeout,verbosity,hpss)
(status,script_stderr)=run_download_script(li,buffered)
killed=is_killed(transfer_protocol,status)
elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GLOBUSTRANSFER:
pass
else:
assert False
return (status,killed,script_stderr)
def replica_next(file_functional_id,args):
import sdrfile, sdmodify, sdfiledao, sdutils, sdconst
parameter=['keep_replica=true','nearest=false','file_functional_id=%s'%file_functional_id]
files=sdrfile.get_files(parameter=parameter,dry_run=args.dry_run)
if not args.dry_run:
if len(files)==0:
print_stderr("File not found in ESGF (file_functional_id=%s)"%file_functional_id)
else:
replicas=[(f['url'],f['data_node']) for f in files]
file_=sdfiledao.get_file(file_functional_id)
if file_ is not None:
if sdutils.get_transfer_protocol(file_.url)==sdconst.TRANSFER_PROTOCOL_HTTP:
sdmodify.replica_next(file_,replicas)
else:
print_stderr("Incorrect protocol") # only http protocol is supported in 'synda replica' for now
else:
print_stderr("Local file not found")
def run(tr):
"""
Returns
True: url has been switched to a new one
False: nothing changed (same url)
"""
transfer_protocol = sdutils.get_transfer_protocol(tr.url)
if transfer_protocol == sdconst.TRANSFER_PROTOCOL_GRIDFTP:
# GRIDFTP url
try:
next_url(tr)
return True
except sdexception.FileNotFoundException as e:
sdlog.info(
"SDNEXTUR-001",
"Cannot switch url for %s (FileNotFoundException)" %
(tr.file_functional_id, ))
return False
except sdexception.HttpUrlNotFoundException as e:
sdlog.info(
"SDNEXTUR-002",
"Cannot switch url for %s (HttpUrlNotFoundException)" %
(tr.file_functional_id, ))
return False
except Exception as e:
sdlog.info(
"SDNEXTUR-003",
"Unknown exception (file_functional_id=%s,exception=%s)" %
(tr.file_functional_id, str(e)))
return False
else:
# most likely HTTP url
#
# nothing to do
# (i.e. no fallback for now in this module for http url)
return False
def local_urls_to_remote_urls(local_urls):
remote_urls=[]
for local_url in local_urls:
protocol=sdutils.get_transfer_protocol(local_url)
gridftp_suffix='|application/gridftp|GridFTP'
opendap_suffix='|application/opendap-html|OPENDAP'
http_suffix='|application/netcdf|HTTPServer'
if protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP:
suffix=gridftp_suffix
elif protocol==sdconst.TRANSFER_PROTOCOL_OPENDAP:
suffix=opendap_suffix
elif protocol==sdconst.TRANSFER_PROTOCOL_HTTP:
suffix=http_suffix
else:
suffix=http_suffix # if unknown, default to HTTP
remote_url='%s%s'%(local_url,suffix)
remote_urls.append(remote_url)
return remote_urls
def local_urls_to_remote_urls(local_urls):
remote_urls = []
for local_url in local_urls:
protocol = sdutils.get_transfer_protocol(local_url)
gridftp_suffix = '|application/gridftp|GridFTP'
opendap_suffix = '|application/opendap-html|OPENDAP'
http_suffix = '|application/netcdf|HTTPServer'
if protocol == sdconst.TRANSFER_PROTOCOL_GRIDFTP:
suffix = gridftp_suffix
elif protocol == sdconst.TRANSFER_PROTOCOL_OPENDAP:
suffix = opendap_suffix
elif protocol == sdconst.TRANSFER_PROTOCOL_HTTP:
suffix = http_suffix
else:
suffix = http_suffix # if unknown, default to HTTP
remote_url = '%s%s' % (local_url, suffix)
remote_urls.append(remote_url)
return remote_urls
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="Download process has been killed"
sdlog.error("SDDMDEFA-190","%s (file_id=%d,url=%s,local_path=%s)"%(tr.error_msg,tr.file_id,tr.url,tr.local_path))
else:
if sdconfig.next_url_on_error:
# Hack
#
# Notes
# - Only active for gridftp url to prevent having useless log message (i.e. there is currently no url switching mecanism for http url)
# - We need a log here so to have a trace of the original failed transfer (i.e. in case the url-switch succeed, the error msg will be reset)
#
transfer_protocol=sdutils.get_transfer_protocol(tr.url)
if transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP:
sdlog.info("SDDMDEFA-088","Transfer failed: try to use another url (%s)"%str(tr))
result=sdnexturl.run(tr)
if result:
tr.status=sdconst.TRANSFER_STATUS_WAITING
tr.error_msg=''
else:
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg='Error occurs during download.'
else:
tr.status=sdconst.TRANSFER_STATUS_ERROR