def dl_2_file(dl_url, fh, block_size=65535, describe=None, **options): """ Download the file with the main url (of Motu) file. Motu can return an error message in the response stream without setting an appropriate http error code. So, in that case, the content-type response is checked, and if it is text/plain, we consider this as an error. dl_url: the complete download url of Motu fh: file handler to use to write the downstream""" stopWatch = stop_watch.localThreadStopWatch() start_time = datetime.datetime.now() log.info("Downloading file (this can take a while)...") # download file temp = open(fh, 'w+b') try: stopWatch.start('processing') m = utils_http.open_url(dl_url, **options) try: # check the real url (after potential redirection) is not a CAS Url scheme match = re.search(utils_cas.CAS_URL_PATTERN, m.url) if match is not None: service, _, _ = dl_url.partition('?') redirection, _, _ = m.url.partition('?') raise Exception( utils_messages.get_external_messages() ['motu-client.exception.authentication.redirected'] % (service, redirection)) # check that content type is not text/plain headers = m.info() if "Content-Type" in headers: if len(headers['Content-Type']) > 0: if not describe: if headers['Content-Type'].startswith( 'text' ) or headers['Content-Type'].find('html') != -1: raise Exception( utils_messages.get_external_messages() ['motu-client.exception.motu.error'] % m.read()) log.info('File type: %s' % headers['Content-Type']) # check if a content length (size of the file) has been send if "Content-Length" in headers: try: # it should be an integer size = int(headers["Content-Length"]) log.info('File size: %s (%i B)' % (utils_unit.convert_bytes(size), size)) except Exception, e: size = -1 log.warn('File size is not an integer: %s' % headers["Content-Length"]) else: size = -1 log.warn('File size: %s' % 'unknown')
def dl_2_file(dl_url, fh, block_size = 65535, describe = 'None', **options): """ Download the file with the main url (of Motu) file. Motu can return an error message in the response stream without setting an appropriate http error code. So, in that case, the content-type response is checked, and if it is text/plain, we consider this as an error. dl_url: the complete download url of Motu fh: file handler to use to write the downstream""" stopWatch = stop_watch.localThreadStopWatch() start_time = datetime.datetime.now() log.info( "Downloading file (this can take a while)..." ) # download file temp = open(fh, 'w+b') try: stopWatch.start('processing') m = utils_http.open_url(dl_url, **options) try: # check the real url (after potential redirection) is not a CAS Url scheme match = re.search(utils_cas.CAS_URL_PATTERN, m.url) if match is not None: service, _, _ = dl_url.partition('?') redirection, _, _ = m.url.partition('?') raise Exception(utils_messages.get_external_messages()['motu-client.exception.authentication.redirected'] % (service, redirection) ) # check that content type is not text/plain headers = m.info() if "Content-Type" in headers: if len(headers['Content-Type']) > 0: if headers['Content-Type'].startswith('text') or headers['Content-Type'].find('html') != -1: raise Exception( utils_messages.get_external_messages()['motu-client.exception.motu.error'] % m.read() ) log.info( 'File type: %s' % headers['Content-Type'] ) # check if a content length (size of the file) has been send if "Content-Length" in headers: try: # it should be an integer size = int(headers["Content-Length"]) log.info( 'File size: %s (%i B)' % ( utils_unit.convert_bytes(size), size ) ) except Exception, e: size = -1 log.warn( 'File size is not an integer: %s' % headers["Content-Length"] ) else: size = -1 log.warn( 'File size: %s' % 'unknown' )
def none_function(sizeRead): percent = 100 log.info("- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent) td = datetime.datetime.now() - start_time
def progress_function(sizeRead): percent = sizeRead * 100. / size log.info("- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent) td = datetime.datetime.now() - start_time
td = datetime.datetime.now() - start_time read = utils_stream.copy( m, temp, progress_function if size != -1 else none_function, block_size) end_time = datetime.datetime.now() stopWatch.stop('downloading') log.info("Processing time : %s", str(processing_time - init_time)) log.info("Downloading time : %s", str(end_time - processing_time)) log.info("Total time : %s", str(end_time - init_time)) log.info( "Download rate : %s/s", utils_unit.convert_bytes( (read / total_milliseconds(end_time - start_time)) * 10**3)) finally: m.close() finally: temp.flush() temp.close() # raise exception if actual size does not match content-length header if size >= 0 and read < size: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.download.too-short'] % (read, size)) def execute_request(_options):
def dl_2_file(dl_url, fh, block_size=65535, isADownloadRequest=None, **options): """ Download the file with the main url (of Motu) file. Motu can return an error message in the response stream without setting an appropriate http error code. So, in that case, the content-type response is checked, and if it is text/plain, we consider this as an error. dl_url: the complete download url of Motu fh: file handler to use to write the downstream""" stopWatch = stop_watch.localThreadStopWatch() start_time = datetime.datetime.now() log.info("Downloading file (this can take a while)...") # download file temp = None if not fh.startswith("console"): temp = open(fh, 'w+b') try: stopWatch.start('processing') m = utils_http.open_url(dl_url, **options) try: # check the real url (after potential redirection) is not a CAS Url scheme match = re.search(utils_cas.CAS_URL_PATTERN, m.url) if match is not None: service, _, _ = dl_url.partition('?') redirection, _, _ = m.url.partition('?') raise Exception( utils_messages.get_external_messages() ['motu-client.exception.authentication.redirected'] % (service, redirection)) # check that content type is not text/plain headers = m.info() if "Content-Type" in headers and len( headers['Content-Type']) > 0 and isADownloadRequest and ( headers['Content-Type'].startswith('text') or headers['Content-Type'].find('html') != -1): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.motu.error'] % m.read()) log.info('File type: %s' % headers['Content-Type']) # check if a content length (size of the file) has been send size = -1 if "Content-Length" in headers: try: # it should be an integer size = int(headers["Content-Length"]) log.info('File size: %s (%i B)' % (utils_unit.convert_bytes(size), size)) except Exception, e: size = -1 log.warn('File size is not an integer: %s' % headers["Content-Length"]) elif temp is not None: log.warn('File size: %s' % 'unknown') processing_time = datetime.datetime.now() stopWatch.stop('processing') stopWatch.start('downloading') # performs the download log.info('Downloading file %s' % os.path.abspath(fh)) def progress_function(sizeRead): percent = sizeRead * 100. / size log.info("- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent) td = datetime.datetime.now() - start_time def none_function(sizeRead): percent = 100 log.info("- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent) td = datetime.datetime.now() - start_time if temp is not None: read = utils_stream.copy( m, temp, progress_function if size != -1 else none_function, block_size) else: if isADownloadRequest: #Console mode, only display the NC file URL on stdout read = len(m.url) print(m.url) else: import cStringIO output = cStringIO.StringIO() utils_stream.copy( m, output, progress_function if size != -1 else none_function, block_size) read = len(output.getvalue()) print(output.getvalue()) end_time = datetime.datetime.now() stopWatch.stop('downloading') log.info("Processing time : %s", str(processing_time - init_time)) log.info("Downloading time : %s", str(end_time - processing_time)) log.info("Total time : %s", str(end_time - init_time)) log.info( "Download rate : %s/s", utils_unit.convert_bytes( (read / total_milliseconds(end_time - start_time)) * 10**3))
def none_function(sizeRead): percent = 100 log.info( "- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent ) td = datetime.datetime.now()- start_time;
def progress_function(sizeRead): percent = sizeRead*100./size log.info( "- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent ) td = datetime.datetime.now()- start_time;
td = datetime.datetime.now()- start_time; def none_function(sizeRead): percent = 100 log.info( "- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent ) td = datetime.datetime.now()- start_time; read = utils_stream.copy(m,temp,progress_function if size != -1 else none_function, block_size ) end_time = datetime.datetime.now() stopWatch.stop('downloading') log.info( "Processing time : %s", str(processing_time - start_time) ) log.info( "Downloading time : %s", str(end_time - processing_time) ) log.info( "Total time : %s", str(end_time - start_time) ) log.info( "Download rate : %s/s", utils_unit.convert_bytes((read / total_milliseconds(end_time - start_time)) * 10**3) ) finally: m.close() finally: temp.flush() temp.close() # raise exception if actual size does not match content-length header if size >= 0 and read < size: raise ContentTooShortError( utils_messages.get_external_messages()['motu-client.exception.download.too-short'] % (read, size), result) def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials)