def end_of_transfer(tr): # log if tr.status==sdconst.TRANSFER_STATUS_DONE: sdlog.info("SDDMDEFA-101","Transfer done (%s)"%str(tr)) elif tr.status==sdconst.TRANSFER_STATUS_WAITING: # Transfer have been marked for retry # (this happens for example during shutdown immediate, where # all running transfers are killed, or when wget are 'stalled' # and killed by watchdog) sdlog.info("SDDMDEFA-108","%s"%(tr.error_msg,)) #sdlog.info("SDDMDEFA-104","Transfer marked for retry (%s)"%str(tr)) else: sdlog.info("SDDMDEFA-102","Transfer failed (%s)"%str(tr)) # update file sdfiledao.update_file(tr) # IMPORTANT: code below must run AFTER the file status has been saved in DB if tr.status==sdconst.TRANSFER_STATUS_DONE: sdevent.file_complete_event(tr) # trigger 'file complete' event # TODO: maybe do some rollback here in case fatal exception occurs in 'file_complete_event' # (else, we have a file marked as 'done' with the corresponding event un-triggered) # check for fatal error if tr.sdget_status==4: sdlog.info("SDDMDEFA-147","Stopping daemon as sdget.download() returned fatal error.") raise FatalException()
def end_of_transfer(tr): # log if tr.status==sdconst.TRANSFER_STATUS_DONE: sdlog.info("SDDMDEFA-101","Transfer done (%s)"%str(tr)) elif tr.status==sdconst.TRANSFER_STATUS_WAITING: # Transfer have been marked for retry # # This may happen for example # - during shutdown immediate, where all running transfers are killed, or when wget are 'stalled' and killed by watchdog # - as a consequence of sdnexturl sdlog.info("SDDMDEFA-108","Transfer marked for retry (error_msg='%s',url=%s,file_id=%d"%(tr.error_msg,tr.url,tr.file_id)) else: sdlog.info("SDDMDEFA-102","Transfer failed (%s)"%str(tr)) # update file sdfiledao.update_file(tr) # IMPORTANT: code below must run AFTER the file status has been saved in DB if tr.status==sdconst.TRANSFER_STATUS_DONE: sdevent.file_complete_event(tr) # trigger 'file complete' event # TODO: maybe do some rollback here in case fatal exception occurs in 'file_complete_event' # (else, we have a file marked as 'done' with the corresponding event un-triggered) # check for fatal error if tr.sdget_status==4: sdlog.info("SDDMDEFA-147","Stopping daemon as sdget.download() returned fatal error.") raise sdexception.FatalException()
def pre_transfer_check_list(tr): """ Return: Check list status True: Check list OK False: Check list NOK """ if lfae_mode == "keep": # usefull mode if # - metadata needs to be regenerated without retransfering the data # - synda files are mixed with files from other sources if os.path.isfile(tr.get_full_local_path()): # file already here, mark the file as done sdlog.info( "SYNDTASK-197", "Local file already exists: keep it (lfae_mode=keep,local_file=%s)" % tr.get_full_local_path()) tr.status = sdconst.TRANSFER_STATUS_DONE tr.error_msg = "Local file already exists: keep it (lfae_mode=keep)" tr.end_date = sdtime.now() sdfiledao.update_file( tr ) # note: it is important not to update a running status in this case, else local file non-related with synda may be removed by synda (because of cleanup_running_transfer() func). See mail from Hans Ramthun at 20150331 for more details. return False else: # file not here, start the download return True elif lfae_mode == "replace": if os.path.isfile(tr.get_full_local_path()): sdlog.info( "SYNDTASK-187", "Local file already exists: remove it (lfae_mode=replace,local_file=%s)" % tr.get_full_local_path()) os.remove(tr.get_full_local_path()) return True elif lfae_mode == "abort": if os.path.isfile(tr.get_full_local_path()): sdlog.info( "SYNDTASK-188", "Local file already exists: transfer aborted (lfae_mode=abort,local_file=%s)" % tr.get_full_local_path()) tr.status = sdconst.TRANSFER_STATUS_ERROR tr.error_msg = "Local file already exists: transfer aborted (lfae_mode=abort)" tr.end_date = sdtime.now() sdfiledao.update_file(tr) return False else: return True
def deferred_delete(file_functional_id): f=sdfiledao.get_file(file_functional_id) f.status=sdconst.TRANSFER_STATUS_DELETE f.error_msg=None f.sdget_status=None f.sdget_error_msg=None sdfiledao.update_file(f)
def deferred_delete(file_functional_id): f = sdfiledao.get_file(file_functional_id) f.status = sdconst.TRANSFER_STATUS_DELETE f.error_msg = None f.sdget_status = None f.sdget_error_msg = None sdfiledao.update_file(f, commit=False)
def start_transfer(tr): """Retrieve next transfer to start Note if no more transfer waiting, get_transfer() raises "NoTransferWaitingException" exception """ def start_transfer_thread(tr): sdfiledao.update_file(tr) th=WorkerThread(tr,eot_queue,Download) th.setDaemon(True) # if main thread quits, we kill running threads (note though that forked child processes are NOT killed and continue running after that !) th.start() # we reset values from previous try if any tr.end_date=None tr.error_msg=None tr.status=sdconst.TRANSFER_STATUS_RUNNING tr.start_date=sdtime.now() if lfae_mode=="keep": # usefull mode if # - metadata needs to be regenerated without retransfering the data # - synda files are mixed with files from other sources if os.path.isfile(tr.get_full_local_path()): # file already here, mark the file as done sdlog.info("SYNDTASK-197","Local file already exists: keep it (lfae_mode=keep,local_file=%s)"%tr.get_full_local_path()) tr.status=sdconst.TRANSFER_STATUS_DONE tr.error_msg="Local file already exists: keep it (lfae_mode=keep)" tr.end_date=sdtime.now() sdfiledao.update_file(tr) # note: it is important not to update a running status in this case, else local file non-related with synda may be removed by synda (because of cleanup_running_transfer() func). See mail from Hans Ramthun at 20150331 for more details. else: # file not here, start the download start_transfer_thread(tr) elif lfae_mode=="replace": if os.path.isfile(tr.get_full_local_path()): sdlog.info("SYNDTASK-187","Local file already exists: remove it (lfae_mode=replace,local_file=%s)"%tr.get_full_local_path()) os.remove(tr.get_full_local_path()) start_transfer_thread(tr) elif lfae_mode=="abort": if os.path.isfile(tr.get_full_local_path()): tr.status=sdconst.TRANSFER_STATUS_ERROR tr.error_msg="Local file already exists: transfer aborted (lfae_mode=abort)" tr.end_date=sdtime.now() sdfiledao.update_file(tr) else: start_transfer_thread(tr)
def pre_transfer_check_list(tr): """ Return: Check list status True: Check list OK False: Check list NOK """ if lfae_mode=="keep": # usefull mode if # - metadata needs to be regenerated without retransfering the data # - synda files are mixed with files from other sources if os.path.isfile(tr.get_full_local_path()): # file already here, mark the file as done sdlog.info("SYNDTASK-197","Local file already exists: keep it (lfae_mode=keep,local_file=%s)"%tr.get_full_local_path()) tr.status=sdconst.TRANSFER_STATUS_DONE tr.error_msg="Local file already exists: keep it (lfae_mode=keep)" tr.end_date=sdtime.now() sdfiledao.update_file(tr) # note: it is important not to update a running status in this case, else local file non-related with synda may be removed by synda (because of cleanup_running_transfer() func). See mail from Hans Ramthun at 20150331 for more details. return False else: # file not here, start the download return True elif lfae_mode=="replace": if os.path.isfile(tr.get_full_local_path()): sdlog.info("SYNDTASK-187","Local file already exists: remove it (lfae_mode=replace,local_file=%s)"%tr.get_full_local_path()) os.remove(tr.get_full_local_path()) return True elif lfae_mode=="abort": if os.path.isfile(tr.get_full_local_path()): sdlog.info("SYNDTASK-188","Local file already exists: transfer aborted (lfae_mode=abort,local_file=%s)"%tr.get_full_local_path()) tr.status=sdconst.TRANSFER_STATUS_ERROR tr.priority -= 1 tr.error_msg="Local file already exists: transfer aborted (lfae_mode=abort)" tr.end_date=sdtime.now() sdfiledao.update_file(tr) return False else: return True
def do_retry(self,arg): if arg=='all': nbr=sdmodify.retry_all() if nbr>0: print "%i file(s) marked for retry."%nbr else: file=sdfiledao.get_file(arg) if file is not None: if file.status == sdconst.TRANSFER_STATUS_ERROR: file.status=sdconst.TRANSFER_STATUS_WAITING sdfiledao.update_file(file) print "File marked for retry" else: print "Retry failed (only file with error status can be retried)." else: print "File not found"
def transfers_begin(): transfers=[] new_transfer_count=max_transfer - sdfilequery.transfer_running_count() # compute how many new transfer can be started if new_transfer_count>0: for i in range(new_transfer_count): try: tr=sddao.get_one_waiting_transfer() prepare_transfer(tr) if pre_transfer_check_list(tr): sdfiledao.update_file(tr) transfers.append(tr) except NoTransferWaitingException, e: pass
def file_(): """This func perform a fake 'end of transfer' event.""" sdlog.info("SDEVENTB-002", "Reset 'end of transfer' events") # check that only files with 'done' status exist li = sdfilequery.get_download_status() if len(li) > 1: raise SDException( 'SDEVENTB-001', "Incorrect files status (status must be 'done' for all files before running this func)" ) # reset files status from done to waiting sdmodifyquery.change_status(sdconst.TRANSFER_STATUS_DONE, sdconst.TRANSFER_STATUS_WAITING) # reset dataset status to empty, and dataset 'latest' flag to false sdmodifyquery.wipeout_datasets_flags(status=sdconst.DATASET_STATUS_EMPTY) # mimic end of transfer dbpagination = sddbpagination.DBPagination() files = dbpagination.get_files() while len(files) > 0: for f in files: sdlog.info("SDEVENTB-003", "trigger eot event on %s" % f.file_functional_id) # PAYLOAD # set status to done f.status = sdconst.TRANSFER_STATUS_DONE sdfiledao.update_file(f) # retrieve the dataset d = sddatasetdao.get_dataset(dataset_id=f.dataset_id) f.dataset = d # trigger end of transfer file event for all files sdevent.file_complete_event(f) sddb.conn.commit() # commit block files = dbpagination.get_files() # next block sdprogress.SDProgressDot.print_char(".")
def transfers_begin(): transfers = [] new_transfer_count = max_transfer - sdfilequery.transfer_running_count( ) # compute how many new transfer can be started if new_transfer_count > 0: for i in range(new_transfer_count): try: tr = sddao.get_one_waiting_transfer() prepare_transfer(tr) if pre_transfer_check_list(tr): sdfiledao.update_file(tr) transfers.append(tr) except NoTransferWaitingException, e: pass
def do_retry(self, arg): import sdfiledao, sdconst, sdmodify if arg == 'all': nbr = sdmodify.retry_all() if nbr > 0: print "%i file(s) marked for retry." % nbr else: file = sdfiledao.get_file(arg) if file is not None: if file.status == sdconst.TRANSFER_STATUS_ERROR: file.status = sdconst.TRANSFER_STATUS_WAITING sdfiledao.update_file(file) print "File marked for retry" else: print "Retry failed (only file with error status can be retried)." else: print "File not found"
def cleanup_running_transfer(): """This handle zombie cases (transfers with 'running' status, but not running). Check for zombie transfer (move "running" transfer to "waiting") Notes: - remaining "running" transfers exist if the daemon has been killed or if the server rebooted when the daemon was running) - if there are still transfers in running state, we switch them to waiting and remove file chunk """ transfer_list=sdfiledao.get_files(status=sdconst.TRANSFER_STATUS_RUNNING) for t in transfer_list: sdlog.info("SDTSCHED-023","fixing transfer status (%s)"%t.get_full_local_path()) if os.path.isfile(t.get_full_local_path()): os.remove(t.get_full_local_path()) t.status=sdconst.TRANSFER_STATUS_WAITING sdfiledao.update_file(t)
def file_(): """This func perform a fake 'end of transfer' event.""" sdlog.info("SDEVENTB-002","Reset 'end of transfer' events") # check that only files with 'done' status exist li=sdfilequery.get_download_status() if len(li)>1: raise SDException('SDEVENTB-001',"Incorrect files status (status must be 'done' for all files before running this func)") # reset files status from done to waiting sdmodifyquery.change_status(sdconst.TRANSFER_STATUS_DONE,sdconst.TRANSFER_STATUS_WAITING) # reset dataset status to empty, and dataset 'latest' flag to false sdmodifyquery.wipeout_datasets_flags(status=sdconst.DATASET_STATUS_EMPTY) # mimic end of transfer dbpagination=sddbpagination.DBPagination() files=dbpagination.get_files() while len(files)>0: for f in files: sdlog.info("SDEVENTB-003","trigger eot event on %s"%f.file_functional_id) # PAYLOAD # set status to done f.status=sdconst.TRANSFER_STATUS_DONE sdfiledao.update_file(f) # retrieve the dataset d=sddatasetdao.get_dataset(dataset_id=f.dataset_id) f.dataset=d # trigger end of transfer file event for all files sdevent.file_complete_event(f) sddb.conn.commit() # commit block files=dbpagination.get_files() # next block sdprogress.SDProgressDot.print_char(".")
def cleanup_running_transfer(): """This handle zombie cases (transfers with 'running' status, but not running). Check for zombie transfer (move "running" transfer to "waiting") Notes: - remaining "running" transfers exist if the daemon has been killed or if the server rebooted when the daemon was running) - if there are still transfers in running state, we switch them to waiting and remove file chunk """ transfer_list = sdfiledao.get_files(status=sdconst.TRANSFER_STATUS_RUNNING) for t in transfer_list: sdlog.info("SDTSCHED-023", "fixing transfer status (%s)" % t.get_full_local_path()) if os.path.isfile(t.get_full_local_path()): os.remove(t.get_full_local_path()) t.status = sdconst.TRANSFER_STATUS_WAITING sdfiledao.update_file(t)
def transfers_begin(): transfers=[] # how many new transfers can be started: new_transfer_count=max_transfer - sdfilequery.transfer_running_count() # datanode_count[datanode], is number of running transfers for a data node: datanode_count = sdfilequery.transfer_running_count_by_datanode() if new_transfer_count>0: transfers_needed = new_transfer_count for i in range(new_transfer_count): for datanode in datanode_count.keys(): try: # Handle per-datanode maximum number of transfers: try: new_count = max_datanode_count - datanode_count[datanode] except KeyError: sdlog.info("SYNDTASK-189","key error on datanode %s, legal keys are %s"% (datanode,datanode_count.keys()) ) new_count = max_datanode_count if new_count<=0: continue tr=sddao.get_one_waiting_transfer( datanode ) prepare_transfer(tr) if pre_transfer_check_list(tr): sdfiledao.update_file(tr) transfers.append(tr) if datanode in datanode_count: datanode_count[datanode] += 1 else: datanode_count[datanode] = 1 transfers_needed -= 1 if transfers_needed <= 0: break except NoTransferWaitingException, e: pass if transfers_needed <= 0: break
os.remove(tr.get_full_local_path()) except Exception, e: sdlog.error( "SDDMGLOB-528", "Error occurs during file suppression (%s,%s)" % (tr.get_full_local_path(), str(e))) elif status == "INACTIVE": # Reactivate both source and destination endpoints activate_endpoint(api, globus_tasks[task_id]['src_endpoint']) activate_endpoint(api) elif status == "ACTIVE": pass # update file sdfiledao.update_file(tr) if tr.status == sdconst.TRANSFER_STATUS_DONE: # TODO: maybe add a try/except and do some rollback here in # case fatal exception occurs in 'file_complete_event' (else, # we have a file marked as 'done' with the corresponding event # un-triggered) # NOTE: code below must run AFTER the file status has been # saved in DB (because it makes DB queries which expect the # file status to exist) sdevent.file_complete_event( tr) # trigger 'file complete' event
if os.path.isfile(tr.get_full_local_path()): try: os.remove(tr.get_full_local_path()) except Exception,e: sdlog.error("SDDMGLOB-528","Error occurs during file suppression (%s,%s)"%(tr.get_full_local_path(),str(e))) elif status == "INACTIVE": # Reactivate both source and destination endpoints activate_endpoint(api, globus_tasks[task_id]['src_endpoint']) activate_endpoint(api) elif status == "ACTIVE": pass # update file sdfiledao.update_file(tr) if tr.status == sdconst.TRANSFER_STATUS_DONE: # TODO: maybe add a try/except and do some rollback here in # case fatal exception occurs in 'file_complete_event' (else, # we have a file marked as 'done' with the corresponding event # un-triggered) # NOTE: code below must run AFTER the file status has been # saved in DB (because it makes DB queries which expect the # file status to exist) sdevent.file_complete_event(tr) # trigger 'file complete' event
def start_transfer_thread(tr): sdfiledao.update_file(tr) th=WorkerThread(tr,eot_queue,Download) th.setDaemon(True) # if main thread quits, we kill running threads (note though that forked child processes are NOT killed and continue running after that !) th.start()