def start_transfer_script(cls,tr):
# renew certificate if needed
try:
sdlogon.renew_certificate(False)
except:
sdlog.error("SDDOWNLO-504","Certificate error: the daemon must be stopped")
raise
checksum_type=tr.checksum_type if tr.checksum_type is not None else 'md5'
(tr.sdget_status,local_checksum,killed,script_stdxxx)=sdget.download(tr.url,tr.get_full_local_path(),checksum_type)
if tr.sdget_status==0:
tr.status=sdconst.TRANSFER_STATUS_DONE
assert tr.size is not None
if int(tr.size) != os.path.getsize(tr.get_full_local_path()):
sdlog.error("SDDOWNLO-002","size don't match (remote_size=%i,local_size=%i,local_path=%s)"%(int(tr.size),os.path.getsize(tr.get_full_local_path()),tr.get_full_local_path()))
# retrieve remote checksum
remote_checksum=tr.checksum
if remote_checksum!=None:
# remote checksum exists
# compare local and remote checksum
if remote_checksum==local_checksum:
# checksum is ok, nothing to do
pass
else:
# checksum is not ok
if incorrect_checksum_action=="remove":
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="File corruption detected: local checksum doesn't match remote checksum"
# remove file from local repository
sdlog.error("SDDOWNLO-155","checksum don't match: remove local file (local_checksum=%s,remote_checksum=%s,local_path=%s)"%(local_checksum,remote_checksum,tr.get_full_local_path()))
try:
os.remove(tr.get_full_local_path())
except Exception,e:
sdlog.error("SDDOWNLO-158","error occurs while removing local file (%s)"%tr.get_full_local_path())
elif incorrect_checksum_action=="keep":
sdlog.info("SDDOWNLO-157","local checksum doesn't match remote checksum (%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
raise SDException("SDDOWNLO-507","incorrect value (%s)"%incorrect_checksum_action)
def test(args):
import os,sdlogon,sdget
sdlogon.renew_certificate(False)
if args.file_url is None:
print_stderr('Incorrect argument')
else:
tmpfile='/tmp/sdt_test_file.nc'
if os.path.isfile(tmpfile):
os.remove(tmpfile)
(sdget_status,local_checksum,killed,script_stderr)=sdget.download(args.file_url,tmpfile,debug=True)
if sdget_status==0:
print_stderr('Transfer completed successfully.')
print_stderr('File location: %s'%tmpfile)
else:
print_stderr(script_stderr)
def test(args):
import os,sdlogon,sdget
sdlogon.renew_certificate(False)
if len(args.parameter)==0:
print_stderr('Incorrect argument')
else:
file_url=args.parameter[0] # it's a naming mess: rename top level action as subcommand
tmpfile='/tmp/sdt_test_file.nc'
if os.path.isfile(tmpfile):
os.remove(tmpfile)
(sdget_status,local_checksum,killed,script_stdxxx)=sdget.download(file_url,tmpfile)
print_stderr(script_stdxxx)
#print_stderr("'Exit code: %i"%sdget_status)
"""
def start_transfer_script(cls,tr):
if sdconfig.fake_download:
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
tr.sdget_error_msg=""
return
(tr.sdget_status,killed,tr.sdget_error_msg)=sdget.download(tr.url,
tr.get_full_local_path(),
debug=False,
http_client=sdconst.HTTP_CLIENT_WGET,
timeout=sdconst.ASYNC_DOWNLOAD_HTTP_TIMEOUT,
verbosity=0,
buffered=True,
hpss=hpss)
if tr.sdget_status==0:
assert tr.size is not None
if int(tr.size) != os.path.getsize(tr.get_full_local_path()):
sdlog.error("SDDMDEFA-002","size don't match (remote_size=%i,local_size=%i,local_path=%s)"%(int(tr.size),os.path.getsize(tr.get_full_local_path()),tr.get_full_local_path()))
# retrieve remote checksum
remote_checksum=tr.checksum
if remote_checksum!=None:
# remote checksum exists
# compute local checksum
checksum_type=tr.checksum_type if tr.checksum_type is not None else sdconst.CHECKSUM_TYPE_MD5 # fallback to 'md5' (arbitrary)
local_checksum=sdutils.compute_checksum(tr.get_full_local_path(),checksum_type)
# compare local and remote checksum
if remote_checksum==local_checksum:
# checksum is ok
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
# checksum is not ok
if incorrect_checksum_action=="remove":
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="File corruption detected: local checksum doesn't match remote checksum"
# remove file from local repository
sdlog.error("SDDMDEFA-155","checksum don't match: remove local file (local_checksum=%s,remote_checksum=%s,local_path=%s)"%(local_checksum,remote_checksum,tr.get_full_local_path()))
try:
os.remove(tr.get_full_local_path())
except Exception,e:
sdlog.error("SDDMDEFA-158","error occurs while removing local file (%s)"%tr.get_full_local_path())
elif incorrect_checksum_action=="keep":
sdlog.info("SDDMDEFA-157","local checksum doesn't match remote checksum (%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
raise sdexception.FatalException("SDDMDEFA-507","incorrect value (%s)"%incorrect_checksum_action)
def run(files,
timeout=sdconst.DIRECT_DOWNLOAD_HTTP_TIMEOUT,
force=False,
http_client=sdconst.HTTP_CLIENT_WGET,
local_path_prefix=sdconfig.sandbox_folder,
verify_checksum=False,
network_bandwidth_test=False,
debug=True,
verbosity=0,
buffered=False,
hpss=False):
"""
Returns:
0 if all transfers complete successfully
1 if one or more transfer(s) didn't complete successfully
"""
"""
If protocol is set to 'globus' in attached_parameters, download all files
with url scheme globus:, or gridftp: provided that there is a gridftp to
globus mapping defined in /esg/config/esgf_endpoints.xml. Remaining files
will be downloaded usingglobus-url-copy or wget.
"""
files = sdglobus.direct(files, force, local_path_prefix, verify_checksum,
network_bandwidth_test, debug, verbosity)
failed_count = 0
for file_ in files:
# check
assert 'url' in file_
#assert 'data_node' in file_
assert 'local_path' in file_
if verify_checksum:
if checksum_attrs_ok(file_):
missing_remote_checksum_attrs = False
else:
missing_remote_checksum_attrs = True
# cast
f = File(**file_)
# prepare attributes
#local_path='/tmp/test.nc'
#local_path='%s/test.nc'%sdconfig.tmp_folder
local_path = f.get_full_local_path(prefix=local_path_prefix)
# check
if not network_bandwidth_test:
if os.path.isfile(local_path):
if force:
os.remove(local_path)
else:
print_stderr(
'Warning: download cancelled as local file already exists (%s)'
% local_path)
failed_count += 1
continue
# special case
if network_bandwidth_test:
local_path = '/dev/null'
# transfer
(status, killed,
script_stderr) = sdget.download(f.url, local_path, debug, http_client,
timeout, verbosity, buffered, hpss)
# post-transfer
if status != 0:
failed_count += 1
print_stderr('Download failed (%s)' % f.url)
if buffered: # in non-buffered mode, stderr is already display (because child stderr is binded to parent stderr)
# currently, we don't come here but we may need in the futur so we keep this block
if script_stderr is not None:
print_stderr(script_stderr)
else:
if network_bandwidth_test:
return
if verify_checksum:
if missing_remote_checksum_attrs:
failed_count += 1
print_stderr(
'Warning: missing remote checksum attributes prevented checksum verification (%s)'
% local_path)
else:
remote_checksum = f.checksum
local_checksum = sdutils.compute_checksum(
local_path, f.checksum_type)
if local_checksum == remote_checksum:
print_stderr(
'File successfully downloaded, checksum OK (%s)' %
local_path)
else:
failed_count += 1
print_stderr(
"Error: local checksum don't match remote checksum (%s)"
% local_path)
else:
print_stderr(
'File successfully downloaded, no checksum verification (%s)'
% local_path)
if failed_count > 0:
return 1
else:
return 0
def start_transfer_script(cls,tr):
sdlog.info("JFPDMDEF-001","Will download url=%s"%(tr.url,))
if sdconfig.fake_download:
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
tr.sdget_error_msg=""
return
# main
(tr.sdget_status,killed,tr.sdget_error_msg)=sdget.download(tr.url,
tr.get_full_local_path(),
debug=False,
http_client=sdconst.HTTP_CLIENT_WGET,
timeout=sdconst.ASYNC_DOWNLOAD_HTTP_TIMEOUT,
verbosity=0,
buffered=True,
hpss=hpss)
# check
assert tr.size is not None
# compute metrics
tr.end_date=sdtime.now()
tr.duration=sdtime.compute_duration(tr.start_date,tr.end_date)
tr.rate=sdtools.compute_rate(tr.size,tr.duration)
# post-processing
if tr.sdget_status==0:
if int(tr.size) != os.path.getsize(tr.get_full_local_path()):
sdlog.error("SDDMDEFA-002","size don't match (remote_size=%i,local_size=%i,local_path=%s)"%(int(tr.size),os.path.getsize(tr.get_full_local_path()),tr.get_full_local_path()))
# retrieve remote checksum
remote_checksum=tr.checksum
if remote_checksum!=None:
# remote checksum exists
# compute local checksum
checksum_type=tr.checksum_type if tr.checksum_type is not None else sdconst.CHECKSUM_TYPE_MD5 # fallback to 'md5' (arbitrary)
local_checksum=sdutils.compute_checksum(tr.get_full_local_path(),checksum_type)
# compare local and remote checksum
if remote_checksum==local_checksum:
# checksum is ok
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
# checksum is not ok
if incorrect_checksum_action=="remove":
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="File corruption detected: local checksum doesn't match remote checksum"
# remove file from local repository
sdlog.error("SDDMDEFA-155","checksum don't match: remove local file (local_checksum=%s,remote_checksum=%s,local_path=%s)"%(local_checksum,remote_checksum,tr.get_full_local_path()))
try:
os.remove(tr.get_full_local_path())
except Exception,e:
sdlog.error("SDDMDEFA-158","error occurs while removing local file (%s)"%tr.get_full_local_path())
elif incorrect_checksum_action=="keep":
sdlog.info("SDDMDEFA-157","local checksum doesn't match remote checksum (%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
raise sdexception.FatalException("SDDMDEFA-507","incorrect value (%s)"%incorrect_checksum_action)
from sdtools import print_stderr
if __name__ == '__main__':
parser = argparse.ArgumentParser()
args = parser.parse_args()
files=json.load( sys.stdin )
for f in files:
assert 'url' in f
assert 'data_node' in f
url=f['url']
dn=f['data_node']
#local_path='/tmp/test.nc'
local_path='%s/test.nc'%sdconfig.tmp_folder
if os.path.isfile(local_path):
os.remove(local_path)
(status,local_checksum,killed,script_stdxxx)=sdget.download(url,local_path,checksum_type='md5',debug=False)
if status!=0:
print_stderr('Download failed: %s'%dn)
else:
print_stderr('File successfully downloaded: %s'%dn)
sys.exit(0)
def run(files,
timeout=sdconst.DIRECT_DOWNLOAD_HTTP_TIMEOUT,
force=False,
http_client=sdconst.HTTP_CLIENT_WGET,
local_path_prefix=sdconfig.sandbox_folder,
verify_checksum=False,
network_bandwidth_test=False,
debug=True,
verbosity=0,
buffered=False,
hpss=False):
"""
Returns:
0 if all transfers complete successfully
1 if one or more transfer(s) didn't complete successfully
"""
failed_count=0
for file_ in files:
# check
assert 'url' in file_
#assert 'data_node' in file_
assert 'local_path' in file_
if verify_checksum:
if checksum_attrs_ok(file_):
missing_remote_checksum_attrs=False
else:
missing_remote_checksum_attrs=True
# cast
f=File(**file_)
# prepare attributes
#local_path='/tmp/test.nc'
#local_path='%s/test.nc'%sdconfig.tmp_folder
local_path=f.get_full_local_path(prefix=local_path_prefix)
# check
if not network_bandwidth_test:
if os.path.isfile(local_path):
if force:
os.remove(local_path)
else:
print_stderr('Warning: download cancelled as local file already exists (%s)'%local_path)
failed_count+=1
continue
# special case
if network_bandwidth_test:
local_path='/dev/null'
# transfer
(status,killed,script_stderr)=sdget.download(f.url,local_path,debug,http_client,timeout,verbosity,buffered,hpss)
# post-transfer
if status!=0:
failed_count+=1
print_stderr('Download failed (%s)'%f.url)
if buffered: # in non-buffered mode, stderr is already display (because child stderr is binded to parent stderr)
# currently, we don't come here but we may need in the futur so we keep this block
if script_stderr is not None:
print_stderr(script_stderr)
else:
if network_bandwidth_test:
return
if verify_checksum:
if missing_remote_checksum_attrs:
failed_count+=1
print_stderr('Warning: missing remote checksum attributes prevented checksum verification (%s)'%local_path)
else:
remote_checksum=f.checksum
local_checksum=sdutils.compute_checksum(local_path,f.checksum_type)
if local_checksum==remote_checksum:
print_stderr('File successfully downloaded, checksum OK (%s)'%local_path)
else:
failed_count+=1
print_stderr("Error: local checksum don't match remote checksum (%s)"%local_path)
else:
print_stderr('File successfully downloaded, no checksum verification (%s)'%local_path)
if failed_count>0:
return 1
else:
return 0