def end_of_transfer(tr):
# log
if tr.status==sdconst.TRANSFER_STATUS_DONE:
sdlog.info("SDDMDEFA-101","Transfer done (%s)"%str(tr))
elif tr.status==sdconst.TRANSFER_STATUS_WAITING:
# Transfer have been marked for retry
# (this happens for example during shutdown immediate, where
# all running transfers are killed, or when wget are 'stalled'
# and killed by watchdog)
sdlog.info("SDDMDEFA-108","%s"%(tr.error_msg,))
#sdlog.info("SDDMDEFA-104","Transfer marked for retry (%s)"%str(tr))
else:
sdlog.info("SDDMDEFA-102","Transfer failed (%s)"%str(tr))
# update file
sdfiledao.update_file(tr)
# IMPORTANT: code below must run AFTER the file status has been saved in DB
if tr.status==sdconst.TRANSFER_STATUS_DONE:
sdevent.file_complete_event(tr) # trigger 'file complete' event
# TODO: maybe do some rollback here in case fatal exception occurs in 'file_complete_event'
# (else, we have a file marked as 'done' with the corresponding event un-triggered)
# check for fatal error
if tr.sdget_status==4:
sdlog.info("SDDMDEFA-147","Stopping daemon as sdget.download() returned fatal error.")
raise FatalException()
def end_of_transfer(tr):
# log
if tr.status==sdconst.TRANSFER_STATUS_DONE:
sdlog.info("SDDMDEFA-101","Transfer done (%s)"%str(tr))
elif tr.status==sdconst.TRANSFER_STATUS_WAITING:
# Transfer have been marked for retry
#
# This may happen for example
# - during shutdown immediate, where all running transfers are killed, or when wget are 'stalled' and killed by watchdog
# - as a consequence of sdnexturl
sdlog.info("SDDMDEFA-108","Transfer marked for retry (error_msg='%s',url=%s,file_id=%d"%(tr.error_msg,tr.url,tr.file_id))
else:
sdlog.info("SDDMDEFA-102","Transfer failed (%s)"%str(tr))
# update file
sdfiledao.update_file(tr)
# IMPORTANT: code below must run AFTER the file status has been saved in DB
if tr.status==sdconst.TRANSFER_STATUS_DONE:
sdevent.file_complete_event(tr) # trigger 'file complete' event
# TODO: maybe do some rollback here in case fatal exception occurs in 'file_complete_event'
# (else, we have a file marked as 'done' with the corresponding event un-triggered)
# check for fatal error
if tr.sdget_status==4:
sdlog.info("SDDMDEFA-147","Stopping daemon as sdget.download() returned fatal error.")
raise sdexception.FatalException()
def pre_transfer_check_list(tr):
"""
Return:
Check list status
True: Check list OK
False: Check list NOK
"""
if lfae_mode == "keep":
# usefull mode if
# - metadata needs to be regenerated without retransfering the data
# - synda files are mixed with files from other sources
if os.path.isfile(tr.get_full_local_path()):
# file already here, mark the file as done
sdlog.info(
"SYNDTASK-197",
"Local file already exists: keep it (lfae_mode=keep,local_file=%s)"
% tr.get_full_local_path())
tr.status = sdconst.TRANSFER_STATUS_DONE
tr.error_msg = "Local file already exists: keep it (lfae_mode=keep)"
tr.end_date = sdtime.now()
sdfiledao.update_file(
tr
) # note: it is important not to update a running status in this case, else local file non-related with synda may be removed by synda (because of cleanup_running_transfer() func). See mail from Hans Ramthun at 20150331 for more details.
return False
else:
# file not here, start the download
return True
elif lfae_mode == "replace":
if os.path.isfile(tr.get_full_local_path()):
sdlog.info(
"SYNDTASK-187",
"Local file already exists: remove it (lfae_mode=replace,local_file=%s)"
% tr.get_full_local_path())
os.remove(tr.get_full_local_path())
return True
elif lfae_mode == "abort":
if os.path.isfile(tr.get_full_local_path()):
sdlog.info(
"SYNDTASK-188",
"Local file already exists: transfer aborted (lfae_mode=abort,local_file=%s)"
% tr.get_full_local_path())
tr.status = sdconst.TRANSFER_STATUS_ERROR
tr.error_msg = "Local file already exists: transfer aborted (lfae_mode=abort)"
tr.end_date = sdtime.now()
sdfiledao.update_file(tr)
return False
else:
return True
def deferred_delete(file_functional_id):
f=sdfiledao.get_file(file_functional_id)
f.status=sdconst.TRANSFER_STATUS_DELETE
f.error_msg=None
f.sdget_status=None
f.sdget_error_msg=None
sdfiledao.update_file(f)
def deferred_delete(file_functional_id):
f = sdfiledao.get_file(file_functional_id)
f.status = sdconst.TRANSFER_STATUS_DELETE
f.error_msg = None
f.sdget_status = None
f.sdget_error_msg = None
sdfiledao.update_file(f, commit=False)
def start_transfer(tr):
"""Retrieve next transfer to start
Note
if no more transfer waiting, get_transfer() raises "NoTransferWaitingException" exception
"""
def start_transfer_thread(tr):
sdfiledao.update_file(tr)
th=WorkerThread(tr,eot_queue,Download)
th.setDaemon(True) # if main thread quits, we kill running threads (note though that forked child processes are NOT killed and continue running after that !)
th.start()
# we reset values from previous try if any
tr.end_date=None
tr.error_msg=None
tr.status=sdconst.TRANSFER_STATUS_RUNNING
tr.start_date=sdtime.now()
if lfae_mode=="keep":
# usefull mode if
# - metadata needs to be regenerated without retransfering the data
# - synda files are mixed with files from other sources
if os.path.isfile(tr.get_full_local_path()):
# file already here, mark the file as done
sdlog.info("SYNDTASK-197","Local file already exists: keep it (lfae_mode=keep,local_file=%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg="Local file already exists: keep it (lfae_mode=keep)"
tr.end_date=sdtime.now()
sdfiledao.update_file(tr) # note: it is important not to update a running status in this case, else local file non-related with synda may be removed by synda (because of cleanup_running_transfer() func). See mail from Hans Ramthun at 20150331 for more details.
else:
# file not here, start the download
start_transfer_thread(tr)
elif lfae_mode=="replace":
if os.path.isfile(tr.get_full_local_path()):
sdlog.info("SYNDTASK-187","Local file already exists: remove it (lfae_mode=replace,local_file=%s)"%tr.get_full_local_path())
os.remove(tr.get_full_local_path())
start_transfer_thread(tr)
elif lfae_mode=="abort":
if os.path.isfile(tr.get_full_local_path()):
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.error_msg="Local file already exists: transfer aborted (lfae_mode=abort)"
tr.end_date=sdtime.now()
sdfiledao.update_file(tr)
else:
start_transfer_thread(tr)
def pre_transfer_check_list(tr):
"""
Return:
Check list status
True: Check list OK
False: Check list NOK
"""
if lfae_mode=="keep":
# usefull mode if
# - metadata needs to be regenerated without retransfering the data
# - synda files are mixed with files from other sources
if os.path.isfile(tr.get_full_local_path()):
# file already here, mark the file as done
sdlog.info("SYNDTASK-197","Local file already exists: keep it (lfae_mode=keep,local_file=%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_DONE
tr.error_msg="Local file already exists: keep it (lfae_mode=keep)"
tr.end_date=sdtime.now()
sdfiledao.update_file(tr) # note: it is important not to update a running status in this case, else local file non-related with synda may be removed by synda (because of cleanup_running_transfer() func). See mail from Hans Ramthun at 20150331 for more details.
return False
else:
# file not here, start the download
return True
elif lfae_mode=="replace":
if os.path.isfile(tr.get_full_local_path()):
sdlog.info("SYNDTASK-187","Local file already exists: remove it (lfae_mode=replace,local_file=%s)"%tr.get_full_local_path())
os.remove(tr.get_full_local_path())
return True
elif lfae_mode=="abort":
if os.path.isfile(tr.get_full_local_path()):
sdlog.info("SYNDTASK-188","Local file already exists: transfer aborted (lfae_mode=abort,local_file=%s)"%tr.get_full_local_path())
tr.status=sdconst.TRANSFER_STATUS_ERROR
tr.priority -= 1
tr.error_msg="Local file already exists: transfer aborted (lfae_mode=abort)"
tr.end_date=sdtime.now()
sdfiledao.update_file(tr)
return False
else:
return True
def do_retry(self,arg):
if arg=='all':
nbr=sdmodify.retry_all()
if nbr>0:
print "%i file(s) marked for retry."%nbr
else:
file=sdfiledao.get_file(arg)
if file is not None:
if file.status == sdconst.TRANSFER_STATUS_ERROR:
file.status=sdconst.TRANSFER_STATUS_WAITING
sdfiledao.update_file(file)
print "File marked for retry"
else:
print "Retry failed (only file with error status can be retried)."
else:
print "File not found"
def transfers_begin():
transfers=[]
new_transfer_count=max_transfer - sdfilequery.transfer_running_count() # compute how many new transfer can be started
if new_transfer_count>0:
for i in range(new_transfer_count):
try:
tr=sddao.get_one_waiting_transfer()
prepare_transfer(tr)
if pre_transfer_check_list(tr):
sdfiledao.update_file(tr)
transfers.append(tr)
except NoTransferWaitingException, e:
pass
def file_():
"""This func perform a fake 'end of transfer' event."""
sdlog.info("SDEVENTB-002", "Reset 'end of transfer' events")
# check that only files with 'done' status exist
li = sdfilequery.get_download_status()
if len(li) > 1:
raise SDException(
'SDEVENTB-001',
"Incorrect files status (status must be 'done' for all files before running this func)"
)
# reset files status from done to waiting
sdmodifyquery.change_status(sdconst.TRANSFER_STATUS_DONE,
sdconst.TRANSFER_STATUS_WAITING)
# reset dataset status to empty, and dataset 'latest' flag to false
sdmodifyquery.wipeout_datasets_flags(status=sdconst.DATASET_STATUS_EMPTY)
# mimic end of transfer
dbpagination = sddbpagination.DBPagination()
files = dbpagination.get_files()
while len(files) > 0:
for f in files:
sdlog.info("SDEVENTB-003",
"trigger eot event on %s" % f.file_functional_id)
# PAYLOAD
# set status to done
f.status = sdconst.TRANSFER_STATUS_DONE
sdfiledao.update_file(f)
# retrieve the dataset
d = sddatasetdao.get_dataset(dataset_id=f.dataset_id)
f.dataset = d
# trigger end of transfer file event for all files
sdevent.file_complete_event(f)
sddb.conn.commit() # commit block
files = dbpagination.get_files() # next block
sdprogress.SDProgressDot.print_char(".")
def transfers_begin():
transfers = []
new_transfer_count = max_transfer - sdfilequery.transfer_running_count(
) # compute how many new transfer can be started
if new_transfer_count > 0:
for i in range(new_transfer_count):
try:
tr = sddao.get_one_waiting_transfer()
prepare_transfer(tr)
if pre_transfer_check_list(tr):
sdfiledao.update_file(tr)
transfers.append(tr)
except NoTransferWaitingException, e:
pass
def do_retry(self, arg):
import sdfiledao, sdconst, sdmodify
if arg == 'all':
nbr = sdmodify.retry_all()
if nbr > 0:
print "%i file(s) marked for retry." % nbr
else:
file = sdfiledao.get_file(arg)
if file is not None:
if file.status == sdconst.TRANSFER_STATUS_ERROR:
file.status = sdconst.TRANSFER_STATUS_WAITING
sdfiledao.update_file(file)
print "File marked for retry"
else:
print "Retry failed (only file with error status can be retried)."
else:
print "File not found"
def cleanup_running_transfer():
"""This handle zombie cases (transfers with 'running' status, but not running).
Check for zombie transfer (move "running" transfer to "waiting")
Notes:
- remaining "running" transfers exist if the daemon has been killed or if the server rebooted when the daemon was running)
- if there are still transfers in running state, we switch them to waiting and remove file chunk
"""
transfer_list=sdfiledao.get_files(status=sdconst.TRANSFER_STATUS_RUNNING)
for t in transfer_list:
sdlog.info("SDTSCHED-023","fixing transfer status (%s)"%t.get_full_local_path())
if os.path.isfile(t.get_full_local_path()):
os.remove(t.get_full_local_path())
t.status=sdconst.TRANSFER_STATUS_WAITING
sdfiledao.update_file(t)
def file_():
"""This func perform a fake 'end of transfer' event."""
sdlog.info("SDEVENTB-002","Reset 'end of transfer' events")
# check that only files with 'done' status exist
li=sdfilequery.get_download_status()
if len(li)>1:
raise SDException('SDEVENTB-001',"Incorrect files status (status must be 'done' for all files before running this func)")
# reset files status from done to waiting
sdmodifyquery.change_status(sdconst.TRANSFER_STATUS_DONE,sdconst.TRANSFER_STATUS_WAITING)
# reset dataset status to empty, and dataset 'latest' flag to false
sdmodifyquery.wipeout_datasets_flags(status=sdconst.DATASET_STATUS_EMPTY)
# mimic end of transfer
dbpagination=sddbpagination.DBPagination()
files=dbpagination.get_files()
while len(files)>0:
for f in files:
sdlog.info("SDEVENTB-003","trigger eot event on %s"%f.file_functional_id)
# PAYLOAD
# set status to done
f.status=sdconst.TRANSFER_STATUS_DONE
sdfiledao.update_file(f)
# retrieve the dataset
d=sddatasetdao.get_dataset(dataset_id=f.dataset_id)
f.dataset=d
# trigger end of transfer file event for all files
sdevent.file_complete_event(f)
sddb.conn.commit() # commit block
files=dbpagination.get_files() # next block
sdprogress.SDProgressDot.print_char(".")
def cleanup_running_transfer():
"""This handle zombie cases (transfers with 'running' status, but not running).
Check for zombie transfer (move "running" transfer to "waiting")
Notes:
- remaining "running" transfers exist if the daemon has been killed or if the server rebooted when the daemon was running)
- if there are still transfers in running state, we switch them to waiting and remove file chunk
"""
transfer_list = sdfiledao.get_files(status=sdconst.TRANSFER_STATUS_RUNNING)
for t in transfer_list:
sdlog.info("SDTSCHED-023",
"fixing transfer status (%s)" % t.get_full_local_path())
if os.path.isfile(t.get_full_local_path()):
os.remove(t.get_full_local_path())
t.status = sdconst.TRANSFER_STATUS_WAITING
sdfiledao.update_file(t)
def transfers_begin():
transfers=[]
# how many new transfers can be started:
new_transfer_count=max_transfer - sdfilequery.transfer_running_count()
# datanode_count[datanode], is number of running transfers for a data node:
datanode_count = sdfilequery.transfer_running_count_by_datanode()
if new_transfer_count>0:
transfers_needed = new_transfer_count
for i in range(new_transfer_count):
for datanode in datanode_count.keys():
try:
# Handle per-datanode maximum number of transfers:
try:
new_count = max_datanode_count - datanode_count[datanode]
except KeyError:
sdlog.info("SYNDTASK-189","key error on datanode %s, legal keys are %s"%
(datanode,datanode_count.keys()) )
new_count = max_datanode_count
if new_count<=0:
continue
tr=sddao.get_one_waiting_transfer( datanode )
prepare_transfer(tr)
if pre_transfer_check_list(tr):
sdfiledao.update_file(tr)
transfers.append(tr)
if datanode in datanode_count:
datanode_count[datanode] += 1
else:
datanode_count[datanode] = 1
transfers_needed -= 1
if transfers_needed <= 0:
break
except NoTransferWaitingException, e:
pass
if transfers_needed <= 0:
break
os.remove(tr.get_full_local_path())
except Exception, e:
sdlog.error(
"SDDMGLOB-528",
"Error occurs during file suppression (%s,%s)" %
(tr.get_full_local_path(), str(e)))
elif status == "INACTIVE":
# Reactivate both source and destination endpoints
activate_endpoint(api, globus_tasks[task_id]['src_endpoint'])
activate_endpoint(api)
elif status == "ACTIVE":
pass
# update file
sdfiledao.update_file(tr)
if tr.status == sdconst.TRANSFER_STATUS_DONE:
# TODO: maybe add a try/except and do some rollback here in
# case fatal exception occurs in 'file_complete_event' (else,
# we have a file marked as 'done' with the corresponding event
# un-triggered)
# NOTE: code below must run AFTER the file status has been
# saved in DB (because it makes DB queries which expect the
# file status to exist)
sdevent.file_complete_event(
tr) # trigger 'file complete' event
if os.path.isfile(tr.get_full_local_path()):
try:
os.remove(tr.get_full_local_path())
except Exception,e:
sdlog.error("SDDMGLOB-528","Error occurs during file suppression (%s,%s)"%(tr.get_full_local_path(),str(e)))
elif status == "INACTIVE":
# Reactivate both source and destination endpoints
activate_endpoint(api, globus_tasks[task_id]['src_endpoint'])
activate_endpoint(api)
elif status == "ACTIVE":
pass
# update file
sdfiledao.update_file(tr)
if tr.status == sdconst.TRANSFER_STATUS_DONE:
# TODO: maybe add a try/except and do some rollback here in
# case fatal exception occurs in 'file_complete_event' (else,
# we have a file marked as 'done' with the corresponding event
# un-triggered)
# NOTE: code below must run AFTER the file status has been
# saved in DB (because it makes DB queries which expect the
# file status to exist)
sdevent.file_complete_event(tr) # trigger 'file complete' event
def start_transfer_thread(tr):
sdfiledao.update_file(tr)
th=WorkerThread(tr,eot_queue,Download)
th.setDaemon(True) # if main thread quits, we kill running threads (note though that forked child processes are NOT killed and continue running after that !)
th.start()
