def run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug_level): if transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP: script=sdconfig.data_download_script_http elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP: script=sdconfig.data_download_script_gridftp else: assert False li=[script,'-c',checksum_type,'-d',str(debug_level),url,full_local_path] (status,stdout,stderr)=sdutils.get_status_output(li,shell=False) # start a new process (fork is blocking here, so thread will wait until child is done) # debug #sdlog.debug("SYNDAGET-002","%s"%stderr) # unexpected errors may be hidden in stderr #sdlog.debug("SYNDAGET-001","%s"%stdout) # unexpected errors may be hidden in stdout if status==0: local_checksum=stdout.rstrip(os.linesep) # if success (status==0), stdout contains only checksum else: local_checksum=None killed=is_killed(transfer_protocol,status) script_stdxxx=stderr # memo: in the child shell script, stdout is redirected to stderr so to leave stdout for the checksum (so stderr contains both stdout and stderr) return (status,local_checksum,killed,script_stdxxx)
def print_certificate(): import os, sdutils if os.path.isfile(sdconfig.esgf_x509_proxy): (sdget_status,stdout,stderr)=sdutils.get_status_output(['/usr/bin/openssl','x509','-in',sdconfig.esgf_x509_proxy,'-text'],shell=False) print stdout else: print_stderr("Certificate not found (use 'renew' command to retrieve a new certificate).")
def certificate_is_valid (): """Checks whether the cert expires in the next 500 seconds.""" li=['/usr/bin/openssl','x509','-checkend','500','-noout','-in',sdconfig.esgf_x509_proxy] (status,stdout,stderr)=sdutils.get_status_output(li,shell=False) if status==0: return True else: return False
def print_certificate(): import os, sdutils certdirprefix=sdconfig.tmp_folder if sdconfig.multiuser else os.environ.get('HOME') certificate_file='%s/.esg/credentials.pem'%certdirprefix if os.path.isfile(certificate_file): (sdget_status,stdout,stderr)=sdutils.get_status_output(['/usr/bin/openssl','x509','-in',certificate_file,'-text'],shell=False) print stdout else: print_stderr("Certificate not found (use 'renew' command to retrieve a new certificate).")
def full_cleanup(): """Remove empty files and folders.""" sdlog.info("SYNCLEAN-008","Starting cleanup in %s."%sdconfig.data_folder) argv=[sdconfig.cleanup_tree_script,sdconfig.data_folder] (status,stdout,stderr)=sdutils.get_status_output(argv) if status!=0: sdtools.trace(sdconfig.stacktrace_log_file,os.path.basename(sdconfig.cleanup_tree_script),status,stdout,stderr) raise SDException("SYNCLEAN-001","Error occurs during tree cleanup") sdlog.info("SYNCLEAN-010","Cleanup done.")
def cleanup_tree(): """Remove empty files and folders.""" sdlog.info("SDOPERAT-008","Starting cleanup in %s."%sdconfig.data_folder) # TODO: this method is only used for incremental mode and it would be usefull to be able to also have a full mode cleanup argv=[sdconfig.cleanup_tree_script,sdconfig.data_folder] (status,stdout,stderr)=sdutils.get_status_output(argv) if status!=0: raise SDException("SDOPERAT-001","Error occurs during tree cleanup (see 'cleanup_tree.log' for details)") sdlog.info("SDOPERAT-010","Cleanup done.")
def certificate_is_valid(): """Checks whether the cert expires in the next 500 seconds.""" li = [ '/usr/bin/openssl', 'x509', '-checkend', '500', '-noout', '-in', sdconfig.esgf_x509_proxy ] (status, stdout, stderr) = sdutils.get_status_output(li, shell=False) if status == 0: return True else: return False
def full_cleanup(): """Remove empty files and folders.""" sdlog.info("SYNCLEAN-008", "Starting cleanup in %s." % sdconfig.data_folder) argv = [sdconfig.cleanup_tree_script, sdconfig.data_folder] (status, stdout, stderr) = sdutils.get_status_output(argv) if status != 0: sdtools.trace(sdconfig.stacktrace_log_file, os.path.basename(sdconfig.cleanup_tree_script), status, stdout, stderr) raise SDException("SYNCLEAN-001", "Error occurs during tree cleanup") sdlog.info("SYNCLEAN-010", "Cleanup done.")
def compute_RTT(remote_host,count=1): """ Args count: how many ping used to compute the average RTT """ rtt=0.0 (status,stdout,stderr)=sdutils.get_status_output('ping -q -c %i %s'%(count,remote_host),shell=True) if status==0: m = re.search('.*min/avg/max/mdev = ([0-9.]+)/([0-9.]+)/([0-9.]+)/([0-9.]+) ms.*', stdout,re.MULTILINE|re.DOTALL) if m: rtt=float(m.group(2)) else: raise SDException("SYNDARTT-001","'ping' output parsing error (%s)"%(stdout,)) else: raise SDException("SYNDARTT-002","'ping' command failed (remote_host=%s,status=%i)"%(remote_host,status,)) return rtt
def renew_certificate(force_renew_certificate,quiet=True,debug=False,force_renew_ca_certificates=False): """Renew ESGF certificate.""" # TODO: move this log into the script so to print only when expired #sdlog.info("SYDLOGON-002","Renew certificate..") (hostname,port,username)=sdopenid.extract_info_from_openid(openid) argv=[sdconfig.logon_script,'-h',hostname,'-p',port,'-s',sdconfig.security_dir,'-u',username] if not quiet: argv.append('-v') if force_renew_certificate: argv.append('-r') if force_renew_ca_certificates: argv.append('-x') (status,stdout,stderr)=sdutils.get_status_output(argv) if status!=0: # print script stdxxx output (useful to debug certificate problem) if quiet: with open(sdconfig.stacktrace_log_file,'a') as fh: fh.write("'%s' script returned an error\n"%os.path.basename(sdconfig.logon_script)) fh.write('status=%s\nstdout=%s\nstderr=%s\n'%(status,stdout.rstrip(os.linesep),stderr.rstrip(os.linesep))) else: print_stderr("'%s' script returned an error\n"%os.path.basename(sdconfig.logon_script)) print_stderr('status=%s\nstdout=%s\nstderr=%s\n'%(status,stdout.rstrip(os.linesep),stderr.rstrip(os.linesep))) sdlog.error("SYDLOGON-040","Exception occured while retrieving certificate (status=%i)"%status) raise CertificateRenewalException("SYDLOGON-001","Cannot retrieve certificate from ESGF (hostname=%s,port=%s)"%(hostname,port)) else: if debug: print_stderr("'%s' script stdxxx (debug mode)\n"%os.path.basename(sdconfig.logon_script)) print_stderr('status=%s\nstdout=%s\nstderr=%s\n'%(status,stdout.rstrip(os.linesep),stderr.rstrip(os.linesep)))
def run_download_script(url,full_local_path,checksum_type,transfer_protocol,debug): if transfer_protocol==sdconst.TRANSFER_PROTOCOL_HTTP: script=sdconfig.data_download_script_http elif transfer_protocol==sdconst.TRANSFER_PROTOCOL_GRIDFTP: script=sdconfig.data_download_script_gridftp else: assert False li=[script,'-c',checksum_type,url,full_local_path] if debug: li.insert(1,'-d') # hpss & parse_wget_output hack hpss=sdconfig.config.getboolean('download','hpss') if hpss: li.insert(1,'-p') li.insert(2,'0') # start a new process (fork is blocking here, so thread will wait until child is done) # # note # in the child shell script, stdout is used for the checksum and stderr for error message # (status,stdout,stderr)=sdutils.get_status_output(li,shell=False) # download scripts may # - return error message on stderr, one line terminated by EOL # - return error message on stderr, multiple lines, each terminated by EOL # first, we chomp # # note # only the last eol is chomped here. # stderr=stderr.rstrip('\r\n') # then we replace all eol from string but the last (which has already been chomped) # stderr=stderr.replace('\n', '<eol-n>').replace('\r', '<eol-r>') # encoding # # Depending on which encoding is set in the system, scripts can return # utf-8, latin1, or something else. # # As for now, Synda encoding is latin1 (this is enforced by setting # 'LANG=C' in external script), we remove any non-latin1 character if any. # # TODO # In the futur, Synda will move to unicode instead of latin1. # Thus, external scripts output encoding will have to be converted to # unicode before being processed by synda # # All synda input should be checked to only accept unicode # (except for specific case where input encoding must be specific, in # which case we make the required explicit conversion to obtain unicode) # # samples # stderr=unicode(stderr, encoding='utf-8') # stderr=unicode(stderr, encoding='latin1') # # stderr=unicode(stderr,errors='ignore').encode('latin1') # debug (unexpected errors may be hidden in stdxxx) """ with open(sdconfig.stacktrace_log_file,'a') as fh: fh.write("BEGIN '%s' script output\n"%os.path.basename(script)) fh.write("status: %s\n"%status) fh.write("stdout:\n") fh.write(stdout) fh.write("stderr:\n") fh.write(stderr) fh.write("END '%s' script output\n"%os.path.basename(script)) """ if status==0: local_checksum=stdout.rstrip(os.linesep) # if success (status==0), stdout contains only checksum else: local_checksum=None killed=is_killed(transfer_protocol,status) return (status,local_checksum,killed,stderr)
def run_download_script_BUFSTDXXX(li): # start a new process (fork is blocking here, so thread will wait until child is done) # # note # in the child shell script, stderr for error message # (status, stdout, stderr) = sdutils.get_status_output(li, shell=False) # download scripts may # - return error message on stderr, one line terminated by EOL # - return error message on stderr, multiple lines, each terminated by EOL # first, we chomp # # note # only the last eol is chomped here. # stderr = stderr.rstrip('\r\n') # then we replace all eol from string but the last (which has already been chomped) # stderr = stderr.replace('\n', '<eol-n>').replace('\r', '<eol-r>') # encoding # # Depending on which encoding is set in the system, scripts can return # utf-8, latin1, or something else. # # As for now, Synda encoding is latin1 (this is enforced by setting # 'LANG=C' in external script), we remove any non-latin1 character if any. # # TODO # In the futur, Synda will move to unicode instead of latin1. # Thus, external scripts output encoding will have to be converted to # unicode before being processed by synda # # All synda input should be checked to only accept unicode # (except for specific case where input encoding must be specific, in # which case we make the required explicit conversion to obtain unicode) # # samples # stderr=unicode(stderr, encoding='utf-8') # stderr=unicode(stderr, encoding='latin1') # # stderr = unicode(stderr, errors='ignore').encode('latin1') # debug (unexpected errors may be hidden in stdxxx) """ with open(sdconfig.stacktrace_log_file,'a') as fh: fh.write("BEGIN '%s' script output\n"%os.path.basename(script)) fh.write("status: %s\n"%status) fh.write("stdout:\n") fh.write(stdout) fh.write("stderr:\n") fh.write(stderr) fh.write("END '%s' script output\n"%os.path.basename(script)) """ return (status, stderr)
def run_download_script_BUFSTDXXX(li): # start a new process (fork is blocking here, so thread will wait until child is done) # # note # in the child shell script, stderr for error message # (status,stdout,stderr)=sdutils.get_status_output(li,shell=False) # download scripts may # - return error message on stderr, one line terminated by EOL # - return error message on stderr, multiple lines, each terminated by EOL # first, we chomp # # note # only the last eol is chomped here. # stderr=stderr.rstrip('\r\n') # then we replace all eol from string but the last (which has already been chomped) # stderr=stderr.replace('\n', '<eol-n>').replace('\r', '<eol-r>') # encoding # # Depending on which encoding is set in the system, scripts can return # utf-8, latin1, or something else. # # As for now, Synda encoding is latin1 (this is enforced by setting # 'LANG=C' in external script), we remove any non-latin1 character if any. # # TODO # In the futur, Synda will move to unicode instead of latin1. # Thus, external scripts output encoding will have to be converted to # unicode before being processed by synda # # All synda input should be checked to only accept unicode # (except for specific case where input encoding must be specific, in # which case we make the required explicit conversion to obtain unicode) # # samples # stderr=unicode(stderr, encoding='utf-8') # stderr=unicode(stderr, encoding='latin1') # # stderr=unicode(stderr,errors='ignore').encode('latin1') # debug (unexpected errors may be hidden in stdxxx) """ with open(sdconfig.stacktrace_log_file,'a') as fh: fh.write("BEGIN '%s' script output\n"%os.path.basename(script)) fh.write("status: %s\n"%status) fh.write("stdout:\n") fh.write(stdout) fh.write("stderr:\n") fh.write(stderr) fh.write("END '%s' script output\n"%os.path.basename(script)) """ return (status,stderr)