def end_of_transfer(tr):
# log
if tr.status==sdconst.TRANSFER_STATUS_DONE:
sdlog.info("SDDMDEFA-101","Transfer done (%s)"%str(tr))
elif tr.status==sdconst.TRANSFER_STATUS_WAITING:
# Transfer have been marked for retry
#
# This may happen for example
# - during shutdown immediate, where all running transfers are killed, or when wget are 'stalled' and killed by watchdog
# - as a consequence of sdnexturl
sdlog.info("SDDMDEFA-108","Transfer marked for retry (error_msg='%s',url=%s,file_id=%d"%(tr.error_msg,tr.url,tr.file_id))
else:
sdlog.info("SDDMDEFA-102","Transfer failed (%s)"%str(tr))
# update file
sdfiledao.update_file(tr)
# IMPORTANT: code below must run AFTER the file status has been saved in DB
if tr.status==sdconst.TRANSFER_STATUS_DONE:
sdevent.file_complete_event(tr) # trigger 'file complete' event
# TODO: maybe do some rollback here in case fatal exception occurs in 'file_complete_event'
# (else, we have a file marked as 'done' with the corresponding event un-triggered)
# check for fatal error
if tr.sdget_status==4:
sdlog.info("SDDMDEFA-147","Stopping daemon as sdget.download() returned fatal error.")
raise sdexception.FatalException()
def start_transfer_script(cls,tr):
if sdconfig.fake_download:
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
tr.sdget_error_msg=""
return
(tr.sdget_status,killed,tr.sdget_error_msg)=sdget.download(tr.url,
tr.get_full_local_path(),
debug=False,
http_client=sdconst.HTTP_CLIENT_WGET,
timeout=sdconst.ASYNC_DOWNLOAD_HTTP_TIMEOUT,
verbosity=0,
buffered=True,
hpss=hpss)
if tr.sdget_status==0:
assert tr.size is not None
if int(tr.size) != os.path.getsize(tr.get_full_local_path()):
sdlog.error("SDDMDEFA-002","size don't match (remote_size=%i,local_size=%i,local_path=%s)"%(int(tr.size),os.path.getsize(tr.get_full_local_path()),tr.get_full_local_path()))
# retrieve remote checksum
remote_checksum=tr.checksum
if remote_checksum!=None:
# remote checksum exists
# compute local checksum
checksum_type=tr.checksum_type if tr.checksum_type is not None else sdconst.CHECKSUM_TYPE_MD5 # fallback to 'md5' (arbitrary)
local_checksum=sdutils.compute_checksum(tr.get_full_local_path(),checksum_type)
# compare local and remote checksum
if remote_checksum==local_checksum:
# checksum is ok
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
# checksum is not ok
if incorrect_checksum_action=="remove":
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="File corruption detected: local checksum doesn't match remote checksum"
# remove file from local repository
sdlog.error("SDDMDEFA-155","checksum don't match: remove local file (local_checksum=%s,remote_checksum=%s,local_path=%s)"%(local_checksum,remote_checksum,tr.get_full_local_path()))
try:
os.remove(tr.get_full_local_path())
except Exception,e:
sdlog.error("SDDMDEFA-158","error occurs while removing local file (%s)"%tr.get_full_local_path())
elif incorrect_checksum_action=="keep":
sdlog.info("SDDMDEFA-157","local checksum doesn't match remote checksum (%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg=""
else:
raise sdexception.FatalException("SDDMDEFA-507","incorrect value (%s)"%incorrect_checksum_action)