def progress_function(sizeRead):
global lastProgressPercentValue
percent = sizeRead * 100. / size
if percent - lastProgressPercentValue > 1 or (
lastProgressPercentValue != 100 and percent >= 100):
log.info("- %s (%.1f%%)",
utils_unit.convert_bytes(size).rjust(8), percent)
lastProgressPercentValue = percent
def none_function(sizeRead):
global lastProgressPercentValue
percent = 100
log.info("- %s (%.1f%%)",
utils_unit.convert_bytes(size).rjust(8), percent)
lastProgressPercentValue = percent
def dl_2_file(dl_url,
fh,
block_size=65535,
isADownloadRequest=None,
**options):
""" Download the file with the main url (of Motu) file.
Motu can return an error message in the response stream without setting an
appropriate http error code. So, in that case, the content-type response is
checked, and if it is text/plain, we consider this as an error.
dl_url: the complete download url of Motu
fh: file handler to use to write the downstream"""
stopWatch = stop_watch.localThreadStopWatch()
start_time = datetime.datetime.now()
lastProgressPercentValue = 0.0
log.info("Downloading file (this can take a while)...")
# download file
temp = None
if not fh.startswith("console"):
temp = open(fh, 'w+b')
try:
stopWatch.start('processing')
m = utils_http.open_url(dl_url, **options)
try:
# check the real url (after potential redirection) is not a CAS Url scheme
match = re.search(utils_cas.CAS_URL_PATTERN, m.url)
if match is not None:
service, _, _ = dl_url.partition('?')
redirection, _, _ = m.url.partition('?')
raise Exception(
utils_messages.get_external_messages()
['motuclient.exception.authentication.redirected'] %
(service, redirection))
# check that content type is not text/plain
headers = m.info()
if "Content-Type" in headers and len(
headers['Content-Type']) > 0 and isADownloadRequest and (
headers['Content-Type'].startswith('text')
or headers['Content-Type'].find('html') != -1):
raise Exception(utils_messages.get_external_messages()
['motuclient.exception.motu.error'] % m.read())
log.info('File type: %s' % headers['Content-Type'])
# check if a content length (size of the file) has been send
size = -1
if "Content-Length" in headers:
try:
# it should be an integer
size = int(headers["Content-Length"])
log.info('File size: %s (%i B)' %
(utils_unit.convert_bytes(size), size))
except Exception as e:
size = -1
log.warn('File size is not an integer: %s' %
headers["Content-Length"])
elif temp is not None:
log.warn('File size: %s' % 'unknown')
processing_time = datetime.datetime.now()
stopWatch.stop('processing')
stopWatch.start('downloading')
# performs the download
log.info('Downloading file %s' % os.path.abspath(fh))
def progress_function(sizeRead):
global lastProgressPercentValue
percent = sizeRead * 100. / size
if percent - lastProgressPercentValue > 1 or (
lastProgressPercentValue != 100 and percent >= 100):
log.info("- %s (%.1f%%)",
utils_unit.convert_bytes(size).rjust(8), percent)
lastProgressPercentValue = percent
def none_function(sizeRead):
global lastProgressPercentValue
percent = 100
log.info("- %s (%.1f%%)",
utils_unit.convert_bytes(size).rjust(8), percent)
lastProgressPercentValue = percent
if temp is not None:
read = utils_stream.copy(
m, temp,
progress_function if size != -1 else none_function,
block_size)
else:
if isADownloadRequest:
#Console mode, only display the NC file URL on stdout
read = len(m.url)
print((m.url))
else:
import io
output = io.StringIO()
utils_stream.copy(
m, output,
progress_function if size != -1 else none_function,
block_size)
read = len(output.getvalue())
print((output.getvalue()))
end_time = datetime.datetime.now()
stopWatch.stop('downloading')
log.info("Processing time : %s", str(processing_time - init_time))
log.info("Downloading time : %s", str(end_time - processing_time))
log.info("Total time : %s", str(end_time - init_time))
log.info(
"Download rate : %s/s",
utils_unit.convert_bytes(
(read / total_milliseconds(end_time - start_time)) *
10**3))
except Exception as e:
log.error("Download failed: %s", e)
if hasattr(e, 'reason'):
log.info(' . reason: %s', e.reason)
if hasattr(e, 'code'):
log.info(' . code %s: ', e.code)
if hasattr(e, 'read'):
try:
log.log(utils_log.TRACE_LEVEL, ' . detail:\n%s', e.read())
except:
pass
log.debug('-' * 60)
log.debug("Stack trace exception is detailed herafter:")
exc_type, exc_value, exc_tb = sys.exc_info()
x = traceback.format_exception(exc_type, exc_value, exc_tb)
for stack in x:
log.debug(' . %s', stack.replace('\n', ''))
log.debug('-' * 60)
log.log(utils_log.TRACE_LEVEL,
'System info is provided hereafter:')
system, node, release, version, machine, processor = platform.uname(
)
log.log(utils_log.TRACE_LEVEL, ' . system : %s', system)
log.log(utils_log.TRACE_LEVEL, ' . node : %s', node)
log.log(utils_log.TRACE_LEVEL, ' . release : %s', release)
log.log(utils_log.TRACE_LEVEL, ' . version : %s', version)
log.log(utils_log.TRACE_LEVEL, ' . machine : %s', machine)
log.log(utils_log.TRACE_LEVEL, ' . processor: %s', processor)
log.log(utils_log.TRACE_LEVEL, ' . python : %s', sys.version)
log.log(utils_log.TRACE_LEVEL, ' . client : %s',
get_client_version())
log.log(utils_log.TRACE_LEVEL, '-' * 60)
finally:
m.close()
finally:
if temp is not None:
temp.flush()
temp.close()
# raise exception if actual size does not match content-length header
if temp is not None and size >= 0 and read < size:
raise Exception(utils_messages.get_external_messages()
['motuclient.exception.download.too-short'] %
(read, size))