def download_single_file(opts, jobnum, fi_idx, fi, status_mon): (source_se_path, target_se_path, local_se_path) = get_fi_path_tuple(opts, fi) show_file_info(jobnum, fi_idx, fi) # Copy files to local folder if not accepted_se(opts, fi): return status_mon.register_file_result(jobnum, fi_idx, 'skipping file on blacklisted SE', FileDownloadStatus.FILE_SE_BLACKLIST) activity_check = Activity('Checking file existance') try: if opts.skip_existing and (se_exists(target_se_path).status(timeout=10, terminate=True) == 0): return status_mon.register_file_result(jobnum, fi_idx, 'skipping already existing file', FileDownloadStatus.FILE_EXISTS) finally: activity_check.finish() if se_exists(os.path.dirname(target_se_path)).status(timeout=10, terminate=True) != 0: activity = Activity('Creating target directory') try: mkdir_proc = se_mkdir(os.path.dirname(target_se_path)) if mkdir_proc.status(timeout=10, terminate=True) != 0: return status_mon.register_file_result(jobnum, fi_idx, 'unable to create target dir', FileDownloadStatus.FILE_MKDIR_FAILED, proc=mkdir_proc) finally: activity.finish() if 'file://' in target_se_path: local_se_path = target_se_path copy_timeout_event = GCEvent() copy_ended_event = GCEvent() monitor_thread = start_daemon('Download monitor %s' % jobnum, download_monitor, jobnum, fi_idx, fi, local_se_path, copy_ended_event, copy_timeout_event) cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path) while (cp_proc.status(timeout=0) is None) and not copy_timeout_event.wait(timeout=0.1): pass copy_ended_event.set() monitor_thread.join() if copy_timeout_event.is_set(): cp_proc.terminate(timeout=1) return status_mon.register_file_result(jobnum, fi_idx, 'Transfer timeout', FileDownloadStatus.FILE_TIMEOUT) elif cp_proc.status(timeout=0, terminate=True) != 0: return status_mon.register_file_result(jobnum, fi_idx, 'Transfer error', FileDownloadStatus.FILE_TIMEOUT, proc=cp_proc) return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)
def _delete(file_se_path, where, what): if se_exists(file_se_path).status(timeout=10, terminate=True) == 0: activity = Activity('Deleting file %s from %s' % (fi[FileInfo.NameDest], where)) rm_proc = se_rm(file_se_path) if rm_proc.status(timeout=60, terminate=True) == 0: log.info(log_intro(jobnum, fi_idx) + 'Deleted file %s', file_se_path) else: log.log_process(rm_proc, msg=log_intro(jobnum, fi_idx) + 'Unable to remove %s' % what) activity.finish()
def download_file(opts, output, jobNum, fileIdx, fileInfo): (hash, _, name_dest, pathSE) = fileInfo output.update_progress(fileIdx) # Copy files to local folder outFilePath = os.path.join(opts.output, name_dest) if opts.selectSE: if not (True in imap(lambda s: s in pathSE, opts.selectSE)): output.error('skip file because it is not located on selected SE!') return if opts.skip_existing and (se_exists(outFilePath).status(timeout=10) == 0): output.error('skip file as it already exists!') return if se_exists(os.path.dirname(outFilePath)).status(timeout=10) != 0: se_mkdir(os.path.dirname(outFilePath)).status(timeout=10) checkPath = 'file:///tmp/dlfs.%s' % name_dest if 'file://' in outFilePath: checkPath = outFilePath if not download_monitored(jobNum, output, fileIdx, checkPath, os.path.join(pathSE, name_dest), outFilePath): return False # Verify => compute md5hash if opts.verify_md5: try: hashLocal = md5sum(checkPath.replace('file://', '')) if not ('file://' in outFilePath): dlfs_rm('file://%s' % checkPath, 'SE file') except KeyboardInterrupt: raise except Exception: hashLocal = None output.update_hash(fileIdx, hashLocal) if hash != hashLocal: return False else: output.update_hash(fileIdx) return True
def download_file(opts, output, jobNum, fileIdx, fileInfo): (hash, _, name_dest, pathSE) = fileInfo output.update_progress(fileIdx) # Copy files to local folder outFilePath = os.path.join(opts.output, name_dest) if opts.selectSE: if not (True in imap(lambda s: s in pathSE, opts.selectSE)): output.error('skip file because it is not located on selected SE!') return if opts.skip_existing and (se_exists(outFilePath).status(timeout = 10) == 0): output.error('skip file as it already exists!') return if se_exists(os.path.dirname(outFilePath)).status(timeout = 10) != 0: se_mkdir(os.path.dirname(outFilePath)).status(timeout = 10) checkPath = 'file:///tmp/dlfs.%s' % name_dest if 'file://' in outFilePath: checkPath = outFilePath if not download_monitored(jobNum, output, fileIdx, checkPath, os.path.join(pathSE, name_dest), outFilePath): return False # Verify => compute md5hash if opts.verify_md5: try: hashLocal = md5sum(checkPath.replace('file://', '')) if not ('file://' in outFilePath): dlfs_rm('file://%s' % checkPath, 'SE file') except KeyboardInterrupt: raise except Exception: hashLocal = None output.update_hash(fileIdx, hashLocal) if hash != hashLocal: return False else: output.update_hash(fileIdx) return True
def cleanup_files(opts, files, failJob, output): for (fileIdx, fileInfo) in enumerate(files): (_, _, name_dest, pathSE) = fileInfo # Remove downloaded files in case of failure if (failJob and opts.rm_local_fail) or (not failJob and opts.rm_local_ok): output.update_status(fileIdx, 'Deleting file %s from local...' % name_dest) outFilePath = os.path.join(opts.output, name_dest) if se_exists(outFilePath).status(timeout = 10) == 0: dlfs_rm(outFilePath, 'local file') # Remove SE files in case of failure if (failJob and opts.rm_se_fail) or (not failJob and opts.rm_se_ok): output.update_status(fileIdx, 'Deleting file %s...' % name_dest) dlfs_rm(os.path.join(pathSE, name_dest), 'SE file') output.update_status(fileIdx, None)
def _delete(file_se_path, where, what): if se_exists(file_se_path).status(timeout=10, terminate=True) == 0: activity = Activity('Deleting file %s from %s' % (fi[FileInfo.NameDest], where)) rm_proc = se_rm(file_se_path) if rm_proc.status(timeout=60, terminate=True) == 0: log.info( log_intro(jobnum, fi_idx) + 'Deleted file %s', file_se_path) else: log.log_process(rm_proc, msg=log_intro(jobnum, fi_idx) + 'Unable to remove %s' % what) activity.finish()
def cleanup_files(opts, files, failJob, output): for (fileIdx, fileInfo) in enumerate(files): (_, _, name_dest, pathSE) = fileInfo # Remove downloaded files in case of failure if (failJob and opts.rm_local_fail) or (not failJob and opts.rm_local_ok): output.update_status(fileIdx, 'Deleting file %s from local...' % name_dest) outFilePath = os.path.join(opts.output, name_dest) if se_exists(outFilePath).status(timeout=10) == 0: dlfs_rm(outFilePath, 'local file') # Remove SE files in case of failure if (failJob and opts.rm_se_fail) or (not failJob and opts.rm_se_ok): output.update_status(fileIdx, 'Deleting file %s...' % name_dest) dlfs_rm(os.path.join(pathSE, name_dest), 'SE file') output.update_status(fileIdx, None)
def download_single_file(opts, jobnum, fi_idx, fi, status_mon): (source_se_path, target_se_path, local_se_path) = get_fi_path_tuple(opts, fi) show_file_info(jobnum, fi_idx, fi) # Copy files to local folder if not accepted_se(opts, fi): return status_mon.register_file_result( jobnum, fi_idx, 'skipping file on blacklisted SE', FileDownloadStatus.FILE_SE_BLACKLIST) activity_check = Activity('Checking file existance') try: if opts.skip_existing and (se_exists(target_se_path).status( timeout=10, terminate=True) == 0): return status_mon.register_file_result( jobnum, fi_idx, 'skipping already existing file', FileDownloadStatus.FILE_EXISTS) finally: activity_check.finish() if se_exists(os.path.dirname(target_se_path)).status(timeout=10, terminate=True) != 0: activity = Activity('Creating target directory') try: mkdir_proc = se_mkdir(os.path.dirname(target_se_path)) if mkdir_proc.status(timeout=10, terminate=True) != 0: return status_mon.register_file_result( jobnum, fi_idx, 'unable to create target dir', FileDownloadStatus.FILE_MKDIR_FAILED, proc=mkdir_proc) finally: activity.finish() if 'file://' in target_se_path: local_se_path = target_se_path copy_timeout_event = GCEvent() copy_ended_event = GCEvent() monitor_thread = start_daemon('Download monitor %s' % jobnum, download_monitor, jobnum, fi_idx, fi, local_se_path, copy_ended_event, copy_timeout_event) cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path) while (cp_proc.status(timeout=0) is None) and not copy_timeout_event.wait(timeout=0.1): pass copy_ended_event.set() monitor_thread.join() if copy_timeout_event.is_set(): cp_proc.terminate(timeout=1) return status_mon.register_file_result(jobnum, fi_idx, 'Transfer timeout', FileDownloadStatus.FILE_TIMEOUT) elif cp_proc.status(timeout=0, terminate=True) != 0: return status_mon.register_file_result(jobnum, fi_idx, 'Transfer error', FileDownloadStatus.FILE_TIMEOUT, proc=cp_proc) return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)