def run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug_level):
if transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP:
script=sdconfig.data_download_script_http
elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP:
script=sdconfig.data_download_script_gridftp
else:
assert False
li=[script,'-c',checksum_type,'-d',str(debug_level),url,full_local_path]
(status,stdout,stderr)=sdutils.get_status_output(li,shell=False) # start a new process (fork is blocking here, so thread will wait until child is done)
# debug
#sdlog.debug("SYNDAGET-002","%s"%stderr) # unexpected errors may be hidden in stderr
#sdlog.debug("SYNDAGET-001","%s"%stdout) # unexpected errors may be hidden in stdout
if status==0:
local_checksum=stdout.rstrip(os.linesep) # if success (status==0), stdout contains only checksum
else:
local_checksum=None
killed=is_killed(transfer_protocol,status)
script_stdxxx=stderr # memo: in the child shell script, stdout is redirected to stderr so to leave stdout for the checksum (so stderr contains both stdout and stderr)
return (status,local_checksum,killed,script_stdxxx)
def print_certificate():
import os, sdutils
if os.path.isfile(sdconfig.esgf_x509_proxy):
(sdget_status,stdout,stderr)=sdutils.get_status_output(['/usr/bin/openssl','x509','-in',sdconfig.esgf_x509_proxy,'-text'],shell=False)
print stdout
else:
print_stderr("Certificate not found (use 'renew' command to retrieve a new certificate).")
def certificate_is_valid ():
"""Checks whether the cert expires in the next 500 seconds."""
li=['/usr/bin/openssl','x509','-checkend','500','-noout','-in',sdconfig.esgf_x509_proxy]
(status,stdout,stderr)=sdutils.get_status_output(li,shell=False)
if status==0:
return True
else:
return False
def print_certificate():
import os, sdutils
certdirprefix=sdconfig.tmp_folder if sdconfig.multiuser else os.environ.get('HOME')
certificate_file='%s/.esg/credentials.pem'%certdirprefix
if os.path.isfile(certificate_file):
(sdget_status,stdout,stderr)=sdutils.get_status_output(['/usr/bin/openssl','x509','-in',certificate_file,'-text'],shell=False)
print stdout
else:
print_stderr("Certificate not found (use 'renew' command to retrieve a new certificate).")
def full_cleanup():
"""Remove empty files and folders."""
sdlog.info("SYNCLEAN-008","Starting cleanup in %s."%sdconfig.data_folder)
argv=[sdconfig.cleanup_tree_script,sdconfig.data_folder]
(status,stdout,stderr)=sdutils.get_status_output(argv)
if status!=0:
sdtools.trace(sdconfig.stacktrace_log_file,os.path.basename(sdconfig.cleanup_tree_script),status,stdout,stderr)
raise SDException("SYNCLEAN-001","Error occurs during tree cleanup")
sdlog.info("SYNCLEAN-010","Cleanup done.")
def cleanup_tree():
"""Remove empty files and folders."""
sdlog.info("SDOPERAT-008","Starting cleanup in %s."%sdconfig.data_folder)
# TODO: this method is only used for incremental mode and it would be usefull to be able to also have a full mode cleanup
argv=[sdconfig.cleanup_tree_script,sdconfig.data_folder]
(status,stdout,stderr)=sdutils.get_status_output(argv)
if status!=0:
raise SDException("SDOPERAT-001","Error occurs during tree cleanup (see 'cleanup_tree.log' for details)")
sdlog.info("SDOPERAT-010","Cleanup done.")
def certificate_is_valid():
"""Checks whether the cert expires in the next 500 seconds."""
li = [
'/usr/bin/openssl', 'x509', '-checkend', '500', '-noout', '-in',
sdconfig.esgf_x509_proxy
]
(status, stdout, stderr) = sdutils.get_status_output(li, shell=False)
if status == 0:
return True
else:
return False
def full_cleanup():
"""Remove empty files and folders."""
sdlog.info("SYNCLEAN-008",
"Starting cleanup in %s." % sdconfig.data_folder)
argv = [sdconfig.cleanup_tree_script, sdconfig.data_folder]
(status, stdout, stderr) = sdutils.get_status_output(argv)
if status != 0:
sdtools.trace(sdconfig.stacktrace_log_file,
os.path.basename(sdconfig.cleanup_tree_script), status,
stdout, stderr)
raise SDException("SYNCLEAN-001", "Error occurs during tree cleanup")
sdlog.info("SYNCLEAN-010", "Cleanup done.")
def compute_RTT(remote_host,count=1):
"""
Args
count: how many ping used to compute the average RTT
"""
rtt=0.0
(status,stdout,stderr)=sdutils.get_status_output('ping -q -c %i %s'%(count,remote_host),shell=True)
if status==0:
m = re.search('.*min/avg/max/mdev = ([0-9.]+)/([0-9.]+)/([0-9.]+)/([0-9.]+) ms.*', stdout,re.MULTILINE|re.DOTALL)
if m:
rtt=float(m.group(2))
else:
raise SDException("SYNDARTT-001","'ping' output parsing error (%s)"%(stdout,))
else:
raise SDException("SYNDARTT-002","'ping' command failed (remote_host=%s,status=%i)"%(remote_host,status,))
return rtt
def renew_certificate(force_renew_certificate,quiet=True,debug=False,force_renew_ca_certificates=False):
"""Renew ESGF certificate."""
# TODO: move this log into the script so to print only when expired
#sdlog.info("SYDLOGON-002","Renew certificate..")
(hostname,port,username)=sdopenid.extract_info_from_openid(openid)
argv=[sdconfig.logon_script,'-h',hostname,'-p',port,'-s',sdconfig.security_dir,'-u',username]
if not quiet:
argv.append('-v')
if force_renew_certificate:
argv.append('-r')
if force_renew_ca_certificates:
argv.append('-x')
(status,stdout,stderr)=sdutils.get_status_output(argv)
if status!=0:
# print script stdxxx output (useful to debug certificate problem)
if quiet:
with open(sdconfig.stacktrace_log_file,'a') as fh:
fh.write("'%s' script returned an error\n"%os.path.basename(sdconfig.logon_script))
fh.write('status=%s\nstdout=%s\nstderr=%s\n'%(status,stdout.rstrip(os.linesep),stderr.rstrip(os.linesep)))
else:
print_stderr("'%s' script returned an error\n"%os.path.basename(sdconfig.logon_script))
print_stderr('status=%s\nstdout=%s\nstderr=%s\n'%(status,stdout.rstrip(os.linesep),stderr.rstrip(os.linesep)))
sdlog.error("SYDLOGON-040","Exception occured while retrieving certificate (status=%i)"%status)
raise CertificateRenewalException("SYDLOGON-001","Cannot retrieve certificate from ESGF (hostname=%s,port=%s)"%(hostname,port))
else:
if debug:
print_stderr("'%s' script stdxxx (debug mode)\n"%os.path.basename(sdconfig.logon_script))
print_stderr('status=%s\nstdout=%s\nstderr=%s\n'%(status,stdout.rstrip(os.linesep),stderr.rstrip(os.linesep)))
def run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug):
if transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP:
script=sdconfig.data_download_script_http
elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP:
script=sdconfig.data_download_script_gridftp
else:
assert False
li=[script,'-c',checksum_type,url,full_local_path]
if debug:
li.insert(1,'-d')
# hpss & parse_wget_output hack
hpss=sdconfig.config.getboolean('download','hpss')
if hpss:
li.insert(1,'-p')
li.insert(2,'0')
# start a new process (fork is blocking here, so thread will wait until child is done)
#
# note
# in the child shell script, stdout is used for the checksum and stderr for error message
#
(status,stdout,stderr)=sdutils.get_status_output(li,shell=False)
# download scripts may
# - return error message on stderr, one line terminated by EOL
# - return error message on stderr, multiple lines, each terminated by EOL
# first, we chomp
#
# note
# only the last eol is chomped here.
#
stderr=stderr.rstrip('\r\n')
# then we replace all eol from string but the last (which has already been chomped)
#
stderr=stderr.replace('\n', '<eol-n>').replace('\r', '<eol-r>')
# encoding
#
# Depending on which encoding is set in the system, scripts can return
# utf-8, latin1, or something else.
#
# As for now, Synda encoding is latin1 (this is enforced by setting
# 'LANG=C' in external script), we remove any non-latin1 character if any.
#
# TODO
# In the futur, Synda will move to unicode instead of latin1.
# Thus, external scripts output encoding will have to be converted to
# unicode before being processed by synda
#
# All synda input should be checked to only accept unicode
# (except for specific case where input encoding must be specific, in
# which case we make the required explicit conversion to obtain unicode)
#
# samples
# stderr=unicode(stderr, encoding='utf-8')
# stderr=unicode(stderr, encoding='latin1')
#
#
stderr=unicode(stderr,errors='ignore').encode('latin1')
# debug (unexpected errors may be hidden in stdxxx)
"""
with open(sdconfig.stacktrace_log_file,'a') as fh:
fh.write("BEGIN '%s' script output\n"%os.path.basename(script))
fh.write("status: %s\n"%status)
fh.write("stdout:\n")
fh.write(stdout)
fh.write("stderr:\n")
fh.write(stderr)
fh.write("END '%s' script output\n"%os.path.basename(script))
"""
if status==0:
local_checksum=stdout.rstrip(os.linesep) # if success (status==0), stdout contains only checksum
else:
local_checksum=None
killed=is_killed(transfer_protocol,status)
return (status,local_checksum,killed,stderr)
def run_download_script_BUFSTDXXX(li):
# start a new process (fork is blocking here, so thread will wait until child is done)
#
# note
# in the child shell script, stderr for error message
#
(status, stdout, stderr) = sdutils.get_status_output(li, shell=False)
# download scripts may
# - return error message on stderr, one line terminated by EOL
# - return error message on stderr, multiple lines, each terminated by EOL
# first, we chomp
#
# note
# only the last eol is chomped here.
#
stderr = stderr.rstrip('\r\n')
# then we replace all eol from string but the last (which has already been chomped)
#
stderr = stderr.replace('\n', '<eol-n>').replace('\r', '<eol-r>')
# encoding
#
# Depending on which encoding is set in the system, scripts can return
# utf-8, latin1, or something else.
#
# As for now, Synda encoding is latin1 (this is enforced by setting
# 'LANG=C' in external script), we remove any non-latin1 character if any.
#
# TODO
# In the futur, Synda will move to unicode instead of latin1.
# Thus, external scripts output encoding will have to be converted to
# unicode before being processed by synda
#
# All synda input should be checked to only accept unicode
# (except for specific case where input encoding must be specific, in
# which case we make the required explicit conversion to obtain unicode)
#
# samples
# stderr=unicode(stderr, encoding='utf-8')
# stderr=unicode(stderr, encoding='latin1')
#
#
stderr = unicode(stderr, errors='ignore').encode('latin1')
# debug (unexpected errors may be hidden in stdxxx)
"""
with open(sdconfig.stacktrace_log_file,'a') as fh:
fh.write("BEGIN '%s' script output\n"%os.path.basename(script))
fh.write("status: %s\n"%status)
fh.write("stdout:\n")
fh.write(stdout)
fh.write("stderr:\n")
fh.write(stderr)
fh.write("END '%s' script output\n"%os.path.basename(script))
"""
return (status, stderr)
def run_download_script_BUFSTDXXX(li):
# start a new process (fork is blocking here, so thread will wait until child is done)
#
# note
# in the child shell script, stderr for error message
#
(status,stdout,stderr)=sdutils.get_status_output(li,shell=False)
# download scripts may
# - return error message on stderr, one line terminated by EOL
# - return error message on stderr, multiple lines, each terminated by EOL
# first, we chomp
#
# note
# only the last eol is chomped here.
#
stderr=stderr.rstrip('\r\n')
# then we replace all eol from string but the last (which has already been chomped)
#
stderr=stderr.replace('\n', '<eol-n>').replace('\r', '<eol-r>')
# encoding
#
# Depending on which encoding is set in the system, scripts can return
# utf-8, latin1, or something else.
#
# As for now, Synda encoding is latin1 (this is enforced by setting
# 'LANG=C' in external script), we remove any non-latin1 character if any.
#
# TODO
# In the futur, Synda will move to unicode instead of latin1.
# Thus, external scripts output encoding will have to be converted to
# unicode before being processed by synda
#
# All synda input should be checked to only accept unicode
# (except for specific case where input encoding must be specific, in
# which case we make the required explicit conversion to obtain unicode)
#
# samples
# stderr=unicode(stderr, encoding='utf-8')
# stderr=unicode(stderr, encoding='latin1')
#
#
stderr=unicode(stderr,errors='ignore').encode('latin1')
# debug (unexpected errors may be hidden in stdxxx)
"""
with open(sdconfig.stacktrace_log_file,'a') as fh:
fh.write("BEGIN '%s' script output\n"%os.path.basename(script))
fh.write("status: %s\n"%status)
fh.write("stdout:\n")
fh.write(stdout)
fh.write("stderr:\n")
fh.write(stderr)
fh.write("END '%s' script output\n"%os.path.basename(script))
"""
return (status,stderr)