def event_loop():
global scheduler_state
sdlog.info("SDTSCHED-533","Connected to %s"%sdconfig.db_file,stderr=True)
scheduler_state=2
start_watchdog()
cleanup_running_transfer()
scheduler_state=1
sdlogon.renew_certificate(True)
sdlog.info("SDTSCHED-902","Transfer daemon is now up and running",stderr=True)
while True:
assert os.path.isfile(sdconfig.daemon_pid_file)
if quit==0:
run_soft_tasks()
run_hard_tasks()
if Download.exception_occurs:
sdlog.error("SDTSCHED-002","Fatal exception occured in download thread",stderr=True)
break
if quit==1:
if can_leave(): # wait until all threads finish and until everything has been processed on the database I/O queue
sdlog.info("SDTSCHED-001","Synchro data service stopped",stderr=True)
break
time.sleep(main_loop_sleep)
print
sdlog.info("SDTSCHED-901","Scheduler successfully stopped",stderr=True)
def transfers_begin(transfers):
# renew certificate if needed
try:
sdlogon.renew_certificate(sdconfig.openid,sdconfig.password,force_renew_certificate=False)
except Exception,e:
sdlog.error("SDDMDEFA-502","Exception occured while retrieving certificate (%s)"%str(e))
raise
def start_transfer_script(cls,tr):
# renew certificate if needed
try:
sdlogon.renew_certificate(False)
except Exception,e:
sdlog.error("SDDMDEFA-502","Exception occured while retrieving certificate (%s)"%str(e))
raise
def transfers_begin(transfers):
# renew certificate if needed
try:
sdlogon.renew_certificate(sdconfig.openid,sdconfig.password,force_renew_certificate=False)
except Exception,e:
sdlog.error("SDDMDEFA-502","Exception occured while retrieving certificate (%s)"%str(e))
raise
def start_transfer_script(cls,tr):
# renew certificate if needed
try:
sdlogon.renew_certificate(False)
except:
sdlog.error("SDDOWNLO-504","Certificate error: the daemon must be stopped")
raise
checksum_type=tr.checksum_type if tr.checksum_type is not None else 'md5'
(tr.sdget_status,local_checksum,killed,script_stdxxx)=sdget.download(tr.url,tr.get_full_local_path(),checksum_type)
if tr.sdget_status==0:
tr.status=sdconst.TRANSFER_STATUS_DONE
assert tr.size is not None
if int(tr.size) != os.path.getsize(tr.get_full_local_path()):
sdlog.error("SDDOWNLO-002","size don't match (remote_size=%i,local_size=%i,local_path=%s)"%(int(tr.size),os.path.getsize(tr.get_full_local_path()),tr.get_full_local_path()))
# retrieve remote checksum
remote_checksum=tr.checksum
if remote_checksum!=None:
# remote checksum exists
# compare local and remote checksum
if remote_checksum==local_checksum:
# checksum is ok, nothing to do
pass
else:
# checksum is not ok
if incorrect_checksum_action=="remove":
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="File corruption detected: local checksum doesn't match remote checksum"
# remove file from local repository
sdlog.error("SDDOWNLO-155","checksum don't match: remove local file (local_checksum=%s,remote_checksum=%s,local_path=%s)"%(local_checksum,remote_checksum,tr.get_full_local_path()))
try:
os.remove(tr.get_full_local_path())
except Exception,e:
sdlog.error("SDDOWNLO-158","error occurs while removing local file (%s)"%tr.get_full_local_path())
elif incorrect_checksum_action=="keep":
sdlog.info("SDDOWNLO-157","local checksum doesn't match remote checksum (%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
raise SDException("SDDOWNLO-507","incorrect value (%s)"%incorrect_checksum_action)
def event_loop():
global scheduler_state
sdlog.info("SDTSCHED-533",
"Connected to %s" % sdconfig.db_file,
stderr=True)
scheduler_state = 2
start_watchdog()
cleanup_running_transfer()
clear_failed_url()
if sdconst.GET_FILES_CACHING:
sdfiledao.highest_waiting_priority(
True, True) #initializes cache of max priorities
scheduler_state = 1
if sdconfig.download:
try:
if sdconfig.is_openid_set():
# In this mode, we keep retrying if ESGF IDP is not accessible (e.g. if ESGF is down)
#
# Note
# To be practical, a 'systemd reload sdt' command must be implemented
# (else, openid change in sdt.conf have no impact until the next
# retry, which may be a few hours..). Because currently, synda is not aware
# of sdt.conf changes while running.
#
#sdlogon.renew_certificate_with_retry(sdconfig.openid,sdconfig.password,force_renew_certificate=True)
#sdlogon.renew_certificate_with_retry_highfreq(sdconfig.openid,sdconfig.password,force_renew_certificate=True)
# In this mode, we stop the daemon if ESGF IDP is not accessible (e.g. if ESGF is down)
#
sdlogon.renew_certificate(sdconfig.openid,
sdconfig.password,
force_renew_certificate=True)
else:
sdlog.error("SDTSCHED-928",
'OpenID not set in configuration file',
stderr=True)
raise OpenIDNotSetException(
"SDTSCHED-264", "OpenID not set in configuration file")
except SDException, e:
sdlog.error("SDTSCHED-920",
"Error occured while retrieving ESGF certificate",
stderr=True)
raise
def certificate(args):
import sdlogon
if args.action is None:
sdlogon.print_certificate()
else:
if args.action=="renew":
if sdlogon.is_openid_set():
try:
sdlogon.renew_certificate(True,quiet=False,debug=False,force_renew_ca_certificates=args.force_renew_ca_certificates)
print_stderr('Certificate successfully renewed.')
except Exception,e:
print_stderr('Error occurs while renewing certificate (%s)'%str(e))
else:
print_stderr('Error: OpenID not set in configuration file.')
elif args.action=="print":
sdlogon.print_certificate()
def certificate(args):
import sdconfig, sdlogon
if args.action is None:
sdlogon.print_certificate()
return 0
else:
if args.action == "renew":
# retrieve openid and passwd
if args.openid and args.password:
# use credential from CLI
oid = args.openid
pwd = args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid = sdconfig.openid
pwd = sdconfig.password
else:
print_stderr(
'Error: OpenID not set in configuration file (%s).' %
sdconfig.credential_file)
return 1
# retrieve certificate
try:
sdlogon.renew_certificate(oid,
pwd,
force_renew_certificate=True,
force_renew_ca_certificates=args.
force_renew_ca_certificates)
print_stderr('Certificate successfully renewed.')
return 0
except Exception, e:
print_stderr('Error occurs while renewing certificate (%s)' %
str(e))
return 1
elif args.action == "info":
print 'ESGF CA certificates location: {}'.format(
sdconfig.esgf_x509_cert_dir)
print 'ESGF user certificate location: {}'.format(
sdconfig.esgf_x509_proxy)
return 0
def certificate(args):
import sdlogon
if len(args.parameter)==0:
sdlogon.print_certificate()
else:
action=args.parameter[0] # it's a naming mess: rename top level action as subcommand
if action=="renew":
if sdlogon.is_openid_set():
try:
sdlogon.renew_certificate(True,quiet=False)
print_stderr('Certificate successfully renewed.')
except Exception,e:
print_stderr('Error occurs while renewing certificate (%s)'%str(e))
else:
print_stderr('Error: OpenID not set in configuration file.')
elif action=="print":
sdlogon.print_certificate()
def test(args):
import os,sdlogon,sdget
sdlogon.renew_certificate(False)
if args.file_url is None:
print_stderr('Incorrect argument')
else:
tmpfile='/tmp/sdt_test_file.nc'
if os.path.isfile(tmpfile):
os.remove(tmpfile)
(sdget_status,local_checksum,killed,script_stderr)=sdget.download(args.file_url,tmpfile,debug=True)
if sdget_status==0:
print_stderr('Transfer completed successfully.')
print_stderr('File location: %s'%tmpfile)
else:
print_stderr(script_stderr)
def event_loop():
global scheduler_state
sdlog.info("SDTSCHED-533","Connected to %s"%sdconfig.db_file,stderr=True)
scheduler_state=2
start_watchdog()
cleanup_running_transfer()
scheduler_state=1
if sdconfig.files_download:
try:
if sdlogon.is_openid_set():
# In this mode, we keep retrying if ESGF IDP is not accessible (e.g. if ESGF is down)
#
# Note
# To be practical, a 'systemd reload sdt' command must be implemented
# (else, openid change in sdt.conf have no impact until the next
# retry, which may be a few hours..). Because currently, synda is not aware
# of sdt.conf changes while running.
#
#sdlogon.renew_certificate_with_retry(True)
#sdlogon.renew_certificate_with_retry_highfreq()
# In this mode, we stop the daemon if ESGF IDP is not accessible (e.g. if ESGF is down)
#
sdlogon.renew_certificate(True)
else:
sdlog.error("SDTSCHED-928",'OpenID not set in configuration file',stderr=True)
raise CertificateRenewalException("SDTSCHED-264","OpenID not set in configuration file")
except SDException,e:
sdlog.error("SDTSCHED-920","Error occured while retrieving ESGF certificate",stderr=True)
raise
def test(args):
import os,sdlogon,sdget
sdlogon.renew_certificate(False)
if len(args.parameter)==0:
print_stderr('Incorrect argument')
else:
file_url=args.parameter[0] # it's a naming mess: rename top level action as subcommand
tmpfile='/tmp/sdt_test_file.nc'
if os.path.isfile(tmpfile):
os.remove(tmpfile)
(sdget_status,local_checksum,killed,script_stdxxx)=sdget.download(file_url,tmpfile)
print_stderr(script_stdxxx)
#print_stderr("'Exit code: %i"%sdget_status)
"""
def certificate(args):
import sdconfig,sdlogon
if args.action is None:
sdlogon.print_certificate()
return 0
else:
if args.action=="renew":
# retrieve openid and passwd
if args.openid and args.password:
# use credential from CLI
oid=args.openid
pwd=args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid=sdconfig.openid
pwd=sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).'%sdconfig.credential_file)
return 1
# retrieve certificate
try:
sdlogon.renew_certificate(oid,pwd,force_renew_certificate=True,force_renew_ca_certificates=args.force_renew_ca_certificates)
print_stderr('Certificate successfully renewed.')
return 0
except Exception,e:
print_stderr('Error occurs while renewing certificate (%s)'%str(e))
return 1
elif args.action=="info":
print 'ESGF CA certificates location: {}'.format(sdconfig.esgf_x509_cert_dir)
print 'ESGF user certificate location: {}'.format(sdconfig.esgf_x509_proxy)
return 0
def get(args):
import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils
# hack
# see TAG43534FSFS
if args.quiet:
args.verbosity=0
if args.verify_checksum and args.network_bandwidth_test:
print_stderr("'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set.")
return 1
stream=syndautils.get_stream(subcommand=args.subcommand,parameter=args.parameter,selection_file=args.selection_file)
if args.openid and args.password:
# use credential from CLI
oid=args.openid
pwd=args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid=sdconfig.openid
pwd=sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).'%sdconfig.credential_file)
return 1
# retrieve certificate
sdlogon.renew_certificate(oid,pwd,force_renew_certificate=False)
http_client=sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET
# local_path
#
# 'synda get' subcommand currently force local_path to the following construct:
# '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may
# change in the future.
#
if args.dest_folder is None:
local_path_prefix=os.getcwd() # current working directory
else:
local_path_prefix=args.dest_folder
# BEWARE
#
# when set in CLI parameter, url is usually an ESGF facet, and as so should
# be sent to the search-api as other facets
# BUT
# we want a special behaviour here (i.e. with 'synda get' command) with url:
# if url is set by user, we DON'T call search-api operator. Instead, we
# download the url directly.
urls=sdearlystreamutils.get_facet_values_early(stream,'url')
if len(urls)==0:
# no url in stream: switch to search-api operator mode
sddeferredafter.add_default_parameter(stream,'limit',5)
sddeferredafter.add_forced_parameter(stream,'local_path_format','notree')
files=sdrfile.get_files(stream=stream,post_pipeline_mode='file',dry_run=args.dry_run) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent
if not args.dry_run:
if len(files)>0:
# compute metric
total_size=sum(int(f['size']) for f in files)
total_size=humanize.naturalsize(total_size,gnu=False)
print_stderr('%i file(s) will be downloaded for a total size of %s.'%(len(files),total_size))
status=sddirectdownload.run(files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum,
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status!=0:
return 1
else:
print_stderr("File not found")
return 1
else:
for f in files:
size=humanize.naturalsize(f['size'],gnu=False)
print '%-12s %s'%(size,f['filename'])
elif len(urls)>0:
# url(s) found in stream: search-api operator not needed (download url directly)
# TAGDSFDF432F
if args.verify_checksum:
print_stderr("To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url.")
return 1
# TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url))
files=[]
for url in urls:
filename=os.path.basename(url)
local_path=filename
f=dict(local_path=local_path,url=url)
files.append(f)
status=sddirectdownload.run(files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status!=0:
return 1
else:
assert False
return 0
def get(args):
import sdlogon, sdrfile, sddeferredafter, sddirectdownload, syndautils, humanize, sdconfig, os, sdconst, sdearlystreamutils
# hack
# see TAG43534FSFS
if args.quiet:
args.verbosity = 0
if args.verify_checksum and args.network_bandwidth_test:
print_stderr(
"'verify_checksum' option cannot be set when 'network_bandwidth_test' option is set."
)
return 1
stream = syndautils.get_stream(subcommand=args.subcommand,
parameter=args.parameter,
selection_file=args.selection_file)
if args.openid and args.password:
# use credential from CLI
oid = args.openid
pwd = args.password
else:
# use credential from file
if sdconfig.is_openid_set():
oid = sdconfig.openid
pwd = sdconfig.password
else:
print_stderr('Error: OpenID not set in configuration file (%s).' %
sdconfig.credential_file)
return 1
# retrieve certificate
sdlogon.renew_certificate(oid, pwd, force_renew_certificate=False)
http_client = sdconst.HTTP_CLIENT_URLLIB if args.urllib2 else sdconst.HTTP_CLIENT_WGET
# local_path
#
# 'synda get' subcommand currently force local_path to the following construct:
# '<dest_folder>/<filename>' (i.e. you can't use DRS tree in-between). This may
# change in the future.
#
if args.dest_folder is None:
local_path_prefix = os.getcwd() # current working directory
else:
local_path_prefix = args.dest_folder
# BEWARE
#
# when set in CLI parameter, url is usually an ESGF facet, and as so should
# be sent to the search-api as other facets
# BUT
# we want a special behaviour here (i.e. with 'synda get' command) with url:
# if url is set by user, we DON'T call search-api operator. Instead, we
# download the url directly.
urls = sdearlystreamutils.get_facet_values_early(stream, 'url')
if len(urls) == 0:
# no url in stream: switch to search-api operator mode
sddeferredafter.add_default_parameter(stream, 'limit', 5)
sddeferredafter.add_forced_parameter(stream, 'local_path_format',
'notree')
files = sdrfile.get_files(
stream=stream, post_pipeline_mode='file', dry_run=args.dry_run
) # yes: this is the second time we run sdinference filter, but it doesn't hurt as sdinference is idempotent
if not args.dry_run:
if len(files) > 0:
# compute metric
total_size = sum(int(f['size']) for f in files)
total_size = humanize.naturalsize(total_size, gnu=False)
print_stderr(
'%i file(s) will be downloaded for a total size of %s.' %
(len(files), total_size))
status = sddirectdownload.run(
files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum,
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status != 0:
return 1
else:
print_stderr("File not found")
return 1
else:
for f in files:
size = humanize.naturalsize(f['size'], gnu=False)
print '%-12s %s' % (size, f['filename'])
elif len(urls) > 0:
# url(s) found in stream: search-api operator not needed (download url directly)
# TAGDSFDF432F
if args.verify_checksum:
print_stderr(
"To perform checksum verification, ESGF file identifier (e.g. title, id, tracking id..) must be used instead of file url."
)
return 1
# TODO: to improve genericity, maybe merge this block into the previous one (i.e. url CAN be used as a search key in the search-api (but not irods url))
files = []
for url in urls:
filename = os.path.basename(url)
local_path = filename
f = dict(local_path=local_path, url=url)
files.append(f)
status = sddirectdownload.run(
files,
args.timeout,
args.force,
http_client,
local_path_prefix,
verify_checksum=args.verify_checksum, # see above at TAGDSFDF432F
network_bandwidth_test=args.network_bandwidth_test,
debug=True,
verbosity=args.verbosity,
buffered=False,
hpss=args.hpss)
if status != 0:
return 1
else:
assert False
return 0