def dl_2_file(dl_url, fh, block_size = 65535, describe = 'None', **options):
""" Download the file with the main url (of Motu) file.
Motu can return an error message in the response stream without setting an
appropriate http error code. So, in that case, the content-type response is
checked, and if it is text/plain, we consider this as an error.
dl_url: the complete download url of Motu
fh: file handler to use to write the downstream"""
stopWatch = stop_watch.localThreadStopWatch()
start_time = datetime.datetime.now()
log.info( "Downloading file (this can take a while)..." )
# download file
temp = open(fh, 'w+b')
try:
stopWatch.start('processing')
m = utils_http.open_url(dl_url, **options)
try:
# check the real url (after potential redirection) is not a CAS Url scheme
match = re.search(utils_cas.CAS_URL_PATTERN, m.url)
if match is not None:
service, _, _ = dl_url.partition('?')
redirection, _, _ = m.url.partition('?')
raise Exception(utils_messages.get_external_messages()['motu-client.exception.authentication.redirected'] % (service, redirection) )
# check that content type is not text/plain
headers = m.info()
if "Content-Type" in headers:
if len(headers['Content-Type']) > 0:
if headers['Content-Type'].startswith('text') or headers['Content-Type'].find('html') != -1:
raise Exception( utils_messages.get_external_messages()['motu-client.exception.motu.error'] % m.read() )
log.info( 'File type: %s' % headers['Content-Type'] )
# check if a content length (size of the file) has been send
if "Content-Length" in headers:
try:
# it should be an integer
size = int(headers["Content-Length"])
log.info( 'File size: %s (%i B)' % ( utils_unit.convert_bytes(size), size ) )
except Exception, e:
size = -1
log.warn( 'File size is not an integer: %s' % headers["Content-Length"] )
else:
size = -1
log.warn( 'File size: %s' % 'unknown' )
def get_requestUrl(dl_url, server, **options):
""" Get the request url."""
stopWatch = stop_watch.localThreadStopWatch()
start_time = datetime.datetime.now()
stopWatch.start('get_request')
log.info( "Requesting file to download (this can take a while)..." )
# Get request id
m = utils_http.open_url(dl_url, **options)
dom = minidom.parseString(m.read())
for node in dom.getElementsByTagName('statusModeResponse'):
requestId = node.getAttribute('requestId')
# Get request url
get_req_url = server + '?action=getreqstatus&requestid=' + requestId
stopWatch.stop('get_request')
return get_req_url
def execute_request(_options):
"""
the main function that submit a request to motu. Available options are:
* Proxy configuration (with eventually user credentials)
- proxy_server: 'http://my-proxy.site.com:8080'
- proxy_user : '******'
- proxy_pwd :'doe'
* Autorisation mode: 'cas', 'basic', 'none'
- auth_mode: 'cas'
* User credentials for authentication 'cas' or 'basic'
- user: '******'
- pwd: 'doe'
* Motu service URL
- motu: 'http://atoll-dev.cls.fr:30080/mis-gateway-servlet/Motu'
* Dataset identifier to download
- product_id: 'dataset-duacs-global-nrt-madt-merged-h'
* Service identifier to use for retrieving dataset
- service_id: 'http://purl.org/myocean/ontology/service/database#yourduname'
* Geographic extraction parameters
- latitude_max : 10.0
- latitude_min : -10.0
- longitude_max: -0.333333333369
- longitude_min: 0.0
* Vertical extraction parameters
- depth_max: 1000
- depth_min: 0
* Temporal extraction parameters, as a datetime instance or a string (format: '%Y-%m-%d %H:%M:%S')
- date_max: 2010-04-25 12:05:36
- date_min: 2010-04-25
* Variable extraction
- variable: ['variable1','variable2']
* The file name and the directory of the downloaded dataset
- out_dir : '.'
- out_name: 'dataset'
* The block size used to perform download
- block_size: 12001
* The socket timeout configuration
- socket_timeout: 515
* The user agent to use when performing http requests
- user_agent: 'motu-api-client'
"""
global log
stopWatch = stop_watch.localThreadStopWatch()
stopWatch.start()
try:
log = logging.getLogger("motu_api")
# at first, we check given options are ok
check_options(_options)
# print some trace info about the options set
log.log( utils_log.TRACE_LEVEL, '-'*60 )
for option in dir(_options):
if not option.startswith('_'):
log.log(utils_log.TRACE_LEVEL, "%s=%s" % (option, getattr( _options, option ) ) )
log.log( utils_log.TRACE_LEVEL, '-'*60 )
# start of url to invoke
url_service = _options.motu
# parameters of the invoked service
url_params = build_params(_options)
url_config = get_url_config(_options)
# check if question mark is in the url
questionMark = '?'
if url_service.endswith(questionMark) :
questionMark = ''
url = url_service+questionMark+url_params
if _options.describe == True:
_options.out_name = _options.out_name.replace('.nc','.xml')
# set-up the socket timeout if any
if _options.socket_timeout != None:
log.debug("Setting timeout %s" % _options.socket_timeout)
socket.setdefaulttimeout(_options.socket_timeout)
if _options.auth_mode == AUTHENTICATION_MODE_CAS:
stopWatch.start('authentication')
# perform authentication before acceding service
download_url = utils_cas.authenticate_CAS_for_URL(url,
_options.user,
_options.pwd,**url_config)
stopWatch.stop('authentication')
else:
# if none, we do nothing more, in basic, we let the url requester doing the job
download_url = url
# create a file for storing downloaded stream
fh = os.path.join(_options.out_dir,_options.out_name)
try:
# Synchronous mode
if _options.sync == True:
dl_2_file(download_url, fh, _options.block_size, **url_config)
log.info( "Done" )
# Asynchronous mode
else:
stopWatch.start('wait_request')
requestUrl = get_requestUrl(download_url, url_service, **url_config)
# asynchronous mode
status = 0
dwurl = ""
while True:
if _options.auth_mode == AUTHENTICATION_MODE_CAS:
stopWatch.start('authentication')
# perform authentication before acceding service
requestUrlCas = utils_cas.authenticate_CAS_for_URL(requestUrl,
_options.user,
_options.pwd,**url_config)
stopWatch.stop('authentication')
else:
# if none, we do nothing more, in basic, we let the url requester doing the job
requestUrlCas = requestUrl
m = utils_http.open_url(requestUrlCas, **url_config)
motu_reply=m.read()
dom = minidom.parseString(motu_reply)
for node in dom.getElementsByTagName('statusModeResponse'):
status = node.getAttribute('status')
dwurl = node.getAttribute('msg')
# Check status
if status == "0" or status == "3": # in progress/pending
log.info('Product is not yet available (request in process)')
time.sleep(10)
else: # finished (error|success)
break
stopWatch.stop('wait_request')
if status == "2": log.error(dwurl)
if status == "1":
log.info('The product is ready for download')
if dwurl != "":
dl_2_file(dwurl, fh, _options.block_size, _options.describe, **url_config)
log.info( "Done" )
else:
log.error("Couldn't retrieve file")
except:
try:
if (os.path.isfile(fh)):
os.remove(fh)
except:
pass
raise
finally:
stopWatch.stop()
def wait_till_finished(reqUrlCAS, **options):
stopWatch = stop_watch.localThreadStopWatch()
start_time = datetime.datetime.now()
def execute_request(_options):
"""
the main function that submit a request to motu. Available options are:
* Proxy configuration (with eventually user credentials)
- proxy_server: 'http://my-proxy.site.com:8080'
- proxy_user : '******'
- proxy_pwd :'doe'
* Autorisation mode: 'cas', 'basic', 'none'
- auth_mode: 'cas'
* User credentials for authentication 'cas' or 'basic'
- user: '******'
- pwd: 'doe'
* Motu service URL
- motu: 'http://atoll-dev.cls.fr:30080/mis-gateway-servlet/Motu'
* Dataset identifier to download
- product_id: 'dataset-duacs-global-nrt-madt-merged-h'
* Service identifier to use for retrieving dataset
- service_id: 'http://purl.org/myocean/ontology/service/database#yourduname'
* Geographic extraction parameters
- latitude_max : 10.0
- latitude_min : -10.0
- longitude_max: -0.333333333369
- longitude_min: 0.0
* Vertical extraction parameters
- depth_max: 1000
- depth_min: 0
* Temporal extraction parameters, as a datetime instance or a string (format: '%Y-%m-%d %H:%M:%S')
- date_max: 2010-04-25 12:05:36
- date_min: 2010-04-25
* Variable extraction
- variable: ['variable1','variable2']
* The file name and the directory of the downloaded dataset
- out_dir : '.'
- out_name: 'dataset'
* The block size used to perform download
- block_size: 12001
* The socket timeout configuration
- socket_timeout: 515
* The user agent to use when performing http requests
- user_agent: 'motu-api-client'
"""
global log
stopWatch = stop_watch.localThreadStopWatch()
stopWatch.start()
try:
log = logging.getLogger("motu_api")
# at first, we check given options are ok
check_options(_options)
# print some trace info about the options set
log.log(utils_log.TRACE_LEVEL, '-' * 60)
for option in dir(_options):
if not option.startswith('_'):
log.log(utils_log.TRACE_LEVEL,
"%s=%s" % (option, getattr(_options, option)))
log.log(utils_log.TRACE_LEVEL, '-' * 60)
# start of url to invoke
url_service = _options.motu
# parameters of the invoked service
url_params = build_params(_options)
url_config = get_url_config(_options)
# check if question mark is in the url
questionMark = '?'
if url_service.endswith(questionMark):
questionMark = ''
url = url_service + questionMark + url_params
# set-up the socket timeout if any
if _options.socket_timeout != None:
log.debug("Setting timeout %s" % _options.socket_timeout)
socket.setdefaulttimeout(_options.socket_timeout)
if _options.auth_mode == AUTHENTICATION_MODE_CAS:
stopWatch.start('authentication')
# perform authentication before acceding service
download_url = utils_cas.authenticate_CAS_for_URL(
url, _options.user, _options.pwd, **url_config)
stopWatch.stop('authentication')
else:
# if none, we do nothing more, in basic, we let the url requester doing the job
download_url = url
# create a file for storing downloaded stream
fh = os.path.join(_options.out_dir, _options.out_name)
try:
dl_2_file(download_url, fh, _options.block_size, **url_config)
log.info("Done")
except:
try:
if (os.path.isfile(fh)):
os.remove(fh)
except:
pass
raise
finally:
stopWatch.stop()
def execute_request(_options):
"""
the main function that submit a request to motu. Available options are:
* Proxy configuration (with eventually user credentials)
- proxy_server: 'http://my-proxy.site.com:8080'
- proxy_user : '******'
- proxy_pwd :'doe'
* Autorisation mode: 'cas', 'basic', 'none'
- auth_mode: 'cas'
* User credentials for authentication 'cas' or 'basic'
- user: '******'
- pwd: 'doe'
* Motu service URL
- motu: 'http://atoll-dev.cls.fr:30080/mis-gateway-servlet/Motu'
* Dataset identifier to download
- product_id: 'dataset-duacs-global-nrt-madt-merged-h'
* Service identifier to use for retrieving dataset
- service_id: 'http://purl.org/myocean/ontology/service/database#yourduname'
* Geographic extraction parameters
- latitude_max : 10.0
- latitude_min : -10.0
- longitude_max: -0.333333333369
- longitude_min: 0.0
* Vertical extraction parameters
- depth_max: 1000
- depth_min: 0
* Temporal extraction parameters, as a datetime instance or a string (format: '%Y-%m-%d %H:%M:%S')
- date_max: 2010-04-25 12:05:36
- date_min: 2010-04-25
* Variable extraction
- variable: ['variable1','variable2']
* The file name and the directory of the downloaded dataset
- out_dir : '.'
- out_name: 'dataset'
* The block size used to perform download
- block_size: 12001
* The socket timeout configuration
- socket_timeout: 515
* The user agent to use when performing http requests
- user_agent: 'motu-api-client'
"""
global log
global init_time
init_time = datetime.datetime.now()
stopWatch = stop_watch.localThreadStopWatch()
stopWatch.start()
try:
log = logging.getLogger("motu_api")
# at first, we check given options are ok
check_options(_options)
# print some trace info about the options set
log.log(utils_log.TRACE_LEVEL, '-' * 60)
for option in dir(_options):
if not option.startswith('_'):
log.log(utils_log.TRACE_LEVEL,
"%s=%s" % (option, getattr(_options, option)))
log.log(utils_log.TRACE_LEVEL, '-' * 60)
# start of url to invoke
url_service = _options.motu
# parameters of the invoked service
url_params = build_params(_options)
url_config = get_url_config(_options)
# check if question mark is in the url
questionMark = '?'
if url_service.endswith(questionMark):
questionMark = ''
url = url_service + questionMark + url_params
if _options.describe == True:
_options.out_name = _options.out_name.replace('.nc', '.xml')
# set-up the socket timeout if any
if _options.socket_timeout != None:
log.debug("Setting timeout %s" % _options.socket_timeout)
socket.setdefaulttimeout(_options.socket_timeout)
if _options.auth_mode == AUTHENTICATION_MODE_CAS:
stopWatch.start('authentication')
# perform authentication before acceding service
download_url = utils_cas.authenticate_CAS_for_URL(
url, _options.user, _options.pwd, **url_config)
stopWatch.stop('authentication')
else:
# if none, we do nothing more, in basic, we let the url requester doing the job
download_url = url
# create a file for storing downloaded stream
fh = os.path.join(_options.out_dir, _options.out_name)
try:
# Synchronous mode
if _options.sync == True:
dl_2_file(download_url, fh, _options.block_size,
_options.describe, **url_config)
log.info("Done")
# Asynchronous mode
else:
stopWatch.start('wait_request')
requestUrl = get_requestUrl(download_url, url_service,
**url_config)
# asynchronous mode
status = 0
dwurl = ""
while True:
if _options.auth_mode == AUTHENTICATION_MODE_CAS:
stopWatch.start('authentication')
# perform authentication before acceding service
requestUrlCas = utils_cas.authenticate_CAS_for_URL(
requestUrl, _options.user, _options.pwd,
**url_config)
stopWatch.stop('authentication')
else:
# if none, we do nothing more, in basic, we let the url requester doing the job
requestUrlCas = requestUrl
m = utils_http.open_url(requestUrlCas, **url_config)
motu_reply = m.read()
dom = minidom.parseString(motu_reply)
for node in dom.getElementsByTagName('statusModeResponse'):
status = node.getAttribute('status')
dwurl = node.getAttribute('remoteUri')
# Check status
if status == "0" or status == "3": # in progress/pending
log.info(
'Product is not yet available (request in process)'
)
time.sleep(10)
else: # finished (error|success)
break
stopWatch.stop('wait_request')
if status == "2": log.error(dwurl)
if status == "1":
log.info('The product is ready for download')
if dwurl != "":
dl_2_file(dwurl, fh, _options.block_size,
_options.describe, **url_config)
log.info("Done")
else:
log.error("Couldn't retrieve file")
except:
try:
if (os.path.isfile(fh)):
os.remove(fh)
except:
pass
raise
finally:
stopWatch.stop()
def wait_till_finished(reqUrlCAS, **options):
stopWatch = stop_watch.localThreadStopWatch()
start_time = datetime.datetime.now()