def download_single_file(opts, jobnum, fi_idx, fi, status_mon):
(source_se_path, target_se_path, local_se_path) = get_fi_path_tuple(opts, fi)
show_file_info(jobnum, fi_idx, fi)
# Copy files to local folder
if not accepted_se(opts, fi):
return status_mon.register_file_result(jobnum, fi_idx, 'skipping file on blacklisted SE',
FileDownloadStatus.FILE_SE_BLACKLIST)
activity_check = Activity('Checking file existance')
try:
if opts.skip_existing and (se_exists(target_se_path).status(timeout=10, terminate=True) == 0):
return status_mon.register_file_result(jobnum, fi_idx, 'skipping already existing file',
FileDownloadStatus.FILE_EXISTS)
finally:
activity_check.finish()
if se_exists(os.path.dirname(target_se_path)).status(timeout=10, terminate=True) != 0:
activity = Activity('Creating target directory')
try:
mkdir_proc = se_mkdir(os.path.dirname(target_se_path))
if mkdir_proc.status(timeout=10, terminate=True) != 0:
return status_mon.register_file_result(jobnum, fi_idx, 'unable to create target dir',
FileDownloadStatus.FILE_MKDIR_FAILED, proc=mkdir_proc)
finally:
activity.finish()
if 'file://' in target_se_path:
local_se_path = target_se_path
copy_timeout_event = GCEvent()
copy_ended_event = GCEvent()
monitor_thread = start_daemon('Download monitor %s' % jobnum, download_monitor,
jobnum, fi_idx, fi, local_se_path, copy_ended_event, copy_timeout_event)
cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path)
while (cp_proc.status(timeout=0) is None) and not copy_timeout_event.wait(timeout=0.1):
pass
copy_ended_event.set()
monitor_thread.join()
if copy_timeout_event.is_set():
cp_proc.terminate(timeout=1)
return status_mon.register_file_result(jobnum, fi_idx, 'Transfer timeout',
FileDownloadStatus.FILE_TIMEOUT)
elif cp_proc.status(timeout=0, terminate=True) != 0:
return status_mon.register_file_result(jobnum, fi_idx, 'Transfer error',
FileDownloadStatus.FILE_TIMEOUT, proc=cp_proc)
return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)
def _delete(file_se_path, where, what):
if se_exists(file_se_path).status(timeout=10, terminate=True) == 0:
activity = Activity('Deleting file %s from %s' % (fi[FileInfo.NameDest], where))
rm_proc = se_rm(file_se_path)
if rm_proc.status(timeout=60, terminate=True) == 0:
log.info(log_intro(jobnum, fi_idx) + 'Deleted file %s', file_se_path)
else:
log.log_process(rm_proc, msg=log_intro(jobnum, fi_idx) + 'Unable to remove %s' % what)
activity.finish()
def download_file(opts, output, jobNum, fileIdx, fileInfo):
(hash, _, name_dest, pathSE) = fileInfo
output.update_progress(fileIdx)
# Copy files to local folder
outFilePath = os.path.join(opts.output, name_dest)
if opts.selectSE:
if not (True in imap(lambda s: s in pathSE, opts.selectSE)):
output.error('skip file because it is not located on selected SE!')
return
if opts.skip_existing and (se_exists(outFilePath).status(timeout=10) == 0):
output.error('skip file as it already exists!')
return
if se_exists(os.path.dirname(outFilePath)).status(timeout=10) != 0:
se_mkdir(os.path.dirname(outFilePath)).status(timeout=10)
checkPath = 'file:///tmp/dlfs.%s' % name_dest
if 'file://' in outFilePath:
checkPath = outFilePath
if not download_monitored(jobNum, output, fileIdx, checkPath,
os.path.join(pathSE, name_dest), outFilePath):
return False
# Verify => compute md5hash
if opts.verify_md5:
try:
hashLocal = md5sum(checkPath.replace('file://', ''))
if not ('file://' in outFilePath):
dlfs_rm('file://%s' % checkPath, 'SE file')
except KeyboardInterrupt:
raise
except Exception:
hashLocal = None
output.update_hash(fileIdx, hashLocal)
if hash != hashLocal:
return False
else:
output.update_hash(fileIdx)
return True
def download_file(opts, output, jobNum, fileIdx, fileInfo):
(hash, _, name_dest, pathSE) = fileInfo
output.update_progress(fileIdx)
# Copy files to local folder
outFilePath = os.path.join(opts.output, name_dest)
if opts.selectSE:
if not (True in imap(lambda s: s in pathSE, opts.selectSE)):
output.error('skip file because it is not located on selected SE!')
return
if opts.skip_existing and (se_exists(outFilePath).status(timeout = 10) == 0):
output.error('skip file as it already exists!')
return
if se_exists(os.path.dirname(outFilePath)).status(timeout = 10) != 0:
se_mkdir(os.path.dirname(outFilePath)).status(timeout = 10)
checkPath = 'file:///tmp/dlfs.%s' % name_dest
if 'file://' in outFilePath:
checkPath = outFilePath
if not download_monitored(jobNum, output, fileIdx, checkPath, os.path.join(pathSE, name_dest), outFilePath):
return False
# Verify => compute md5hash
if opts.verify_md5:
try:
hashLocal = md5sum(checkPath.replace('file://', ''))
if not ('file://' in outFilePath):
dlfs_rm('file://%s' % checkPath, 'SE file')
except KeyboardInterrupt:
raise
except Exception:
hashLocal = None
output.update_hash(fileIdx, hashLocal)
if hash != hashLocal:
return False
else:
output.update_hash(fileIdx)
return True
def cleanup_files(opts, files, failJob, output): for (fileIdx, fileInfo) in enumerate(files): (_, _, name_dest, pathSE) = fileInfo # Remove downloaded files in case of failure if (failJob and opts.rm_local_fail) or (not failJob and opts.rm_local_ok): output.update_status(fileIdx, 'Deleting file %s from local...' % name_dest) outFilePath = os.path.join(opts.output, name_dest) if se_exists(outFilePath).status(timeout = 10) == 0: dlfs_rm(outFilePath, 'local file') # Remove SE files in case of failure if (failJob and opts.rm_se_fail) or (not failJob and opts.rm_se_ok): output.update_status(fileIdx, 'Deleting file %s...' % name_dest) dlfs_rm(os.path.join(pathSE, name_dest), 'SE file') output.update_status(fileIdx, None)
def _delete(file_se_path, where, what):
if se_exists(file_se_path).status(timeout=10, terminate=True) == 0:
activity = Activity('Deleting file %s from %s' %
(fi[FileInfo.NameDest], where))
rm_proc = se_rm(file_se_path)
if rm_proc.status(timeout=60, terminate=True) == 0:
log.info(
log_intro(jobnum, fi_idx) + 'Deleted file %s',
file_se_path)
else:
log.log_process(rm_proc,
msg=log_intro(jobnum, fi_idx) +
'Unable to remove %s' % what)
activity.finish()
def cleanup_files(opts, files, failJob, output):
for (fileIdx, fileInfo) in enumerate(files):
(_, _, name_dest, pathSE) = fileInfo
# Remove downloaded files in case of failure
if (failJob and opts.rm_local_fail) or (not failJob
and opts.rm_local_ok):
output.update_status(fileIdx,
'Deleting file %s from local...' % name_dest)
outFilePath = os.path.join(opts.output, name_dest)
if se_exists(outFilePath).status(timeout=10) == 0:
dlfs_rm(outFilePath, 'local file')
# Remove SE files in case of failure
if (failJob and opts.rm_se_fail) or (not failJob and opts.rm_se_ok):
output.update_status(fileIdx, 'Deleting file %s...' % name_dest)
dlfs_rm(os.path.join(pathSE, name_dest), 'SE file')
output.update_status(fileIdx, None)
def download_single_file(opts, jobnum, fi_idx, fi, status_mon):
(source_se_path, target_se_path,
local_se_path) = get_fi_path_tuple(opts, fi)
show_file_info(jobnum, fi_idx, fi)
# Copy files to local folder
if not accepted_se(opts, fi):
return status_mon.register_file_result(
jobnum, fi_idx, 'skipping file on blacklisted SE',
FileDownloadStatus.FILE_SE_BLACKLIST)
activity_check = Activity('Checking file existance')
try:
if opts.skip_existing and (se_exists(target_se_path).status(
timeout=10, terminate=True) == 0):
return status_mon.register_file_result(
jobnum, fi_idx, 'skipping already existing file',
FileDownloadStatus.FILE_EXISTS)
finally:
activity_check.finish()
if se_exists(os.path.dirname(target_se_path)).status(timeout=10,
terminate=True) != 0:
activity = Activity('Creating target directory')
try:
mkdir_proc = se_mkdir(os.path.dirname(target_se_path))
if mkdir_proc.status(timeout=10, terminate=True) != 0:
return status_mon.register_file_result(
jobnum,
fi_idx,
'unable to create target dir',
FileDownloadStatus.FILE_MKDIR_FAILED,
proc=mkdir_proc)
finally:
activity.finish()
if 'file://' in target_se_path:
local_se_path = target_se_path
copy_timeout_event = GCEvent()
copy_ended_event = GCEvent()
monitor_thread = start_daemon('Download monitor %s' % jobnum,
download_monitor, jobnum, fi_idx, fi,
local_se_path, copy_ended_event,
copy_timeout_event)
cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path)
while (cp_proc.status(timeout=0) is
None) and not copy_timeout_event.wait(timeout=0.1):
pass
copy_ended_event.set()
monitor_thread.join()
if copy_timeout_event.is_set():
cp_proc.terminate(timeout=1)
return status_mon.register_file_result(jobnum, fi_idx,
'Transfer timeout',
FileDownloadStatus.FILE_TIMEOUT)
elif cp_proc.status(timeout=0, terminate=True) != 0:
return status_mon.register_file_result(jobnum,
fi_idx,
'Transfer error',
FileDownloadStatus.FILE_TIMEOUT,
proc=cp_proc)
return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)