def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials) - proxy_server: 'http://my-proxy.site.com:8080' - proxy_user : '******' - proxy_pwd :'doe' * Autorisation mode: 'cas', 'basic', 'none' - auth_mode: 'cas' * User credentials for authentication 'cas' or 'basic' - user: '******' - pwd: 'doe' * Motu service URL - motu: 'http://atoll-dev.cls.fr:30080/mis-gateway-servlet/Motu' * Dataset identifier to download - product_id: 'dataset-duacs-global-nrt-madt-merged-h' * Service identifier to use for retrieving dataset - service_id: 'http://purl.org/myocean/ontology/service/database#yourduname' * Geographic extraction parameters - latitude_max : 10.0 - latitude_min : -10.0 - longitude_max: -0.333333333369 - longitude_min: 0.0 * Vertical extraction parameters - depth_max: 1000 - depth_min: 0 * Temporal extraction parameters, as a datetime instance or a string (format: '%Y-%m-%d %H:%M:%S') - date_max: 2010-04-25 12:05:36 - date_min: 2010-04-25 * Variable extraction - variable: ['variable1','variable2'] * The file name and the directory of the downloaded dataset - out_dir : '.' - out_name: 'dataset' * The block size used to perform download - block_size: 12001 * The socket timeout configuration - socket_timeout: 515 * The user agent to use when performing http requests - user_agent: 'motu-api-client' """ global log stopWatch = stop_watch.localThreadStopWatch() stopWatch.start() try: log = logging.getLogger("motu_api") # at first, we check given options are ok check_options(_options) # print some trace info about the options set log.log( utils_log.TRACE_LEVEL, '-'*60 ) for option in dir(_options): if not option.startswith('_'): log.log(utils_log.TRACE_LEVEL, "%s=%s" % (option, getattr( _options, option ) ) ) log.log( utils_log.TRACE_LEVEL, '-'*60 ) # start of url to invoke url_service = _options.motu # parameters of the invoked service url_params = build_params(_options) url_config = get_url_config(_options) # check if question mark is in the url questionMark = '?' if url_service.endswith(questionMark) : questionMark = '' url = url_service+questionMark+url_params if _options.describe == True: _options.out_name = _options.out_name.replace('.nc','.xml') # set-up the socket timeout if any if _options.socket_timeout != None: log.debug("Setting timeout %s" % _options.socket_timeout) socket.setdefaulttimeout(_options.socket_timeout) if _options.auth_mode == AUTHENTICATION_MODE_CAS: stopWatch.start('authentication') # perform authentication before acceding service download_url = utils_cas.authenticate_CAS_for_URL(url, _options.user, _options.pwd,**url_config) stopWatch.stop('authentication') else: # if none, we do nothing more, in basic, we let the url requester doing the job download_url = url # create a file for storing downloaded stream fh = os.path.join(_options.out_dir,_options.out_name) try: # Synchronous mode if _options.sync == True: dl_2_file(download_url, fh, _options.block_size, **url_config) log.info( "Done" ) # Asynchronous mode else: stopWatch.start('wait_request') requestUrl = get_requestUrl(download_url, url_service, **url_config) # asynchronous mode status = 0 dwurl = "" while True: if _options.auth_mode == AUTHENTICATION_MODE_CAS: stopWatch.start('authentication') # perform authentication before acceding service requestUrlCas = utils_cas.authenticate_CAS_for_URL(requestUrl, _options.user, _options.pwd,**url_config) stopWatch.stop('authentication') else: # if none, we do nothing more, in basic, we let the url requester doing the job requestUrlCas = requestUrl m = utils_http.open_url(requestUrlCas, **url_config) motu_reply=m.read() dom = minidom.parseString(motu_reply) for node in dom.getElementsByTagName('statusModeResponse'): status = node.getAttribute('status') dwurl = node.getAttribute('msg') # Check status if status == "0" or status == "3": # in progress/pending log.info('Product is not yet available (request in process)') time.sleep(10) else: # finished (error|success) break stopWatch.stop('wait_request') if status == "2": log.error(dwurl) if status == "1": log.info('The product is ready for download') if dwurl != "": dl_2_file(dwurl, fh, _options.block_size, _options.describe, **url_config) log.info( "Done" ) else: log.error("Couldn't retrieve file") except: try: if (os.path.isfile(fh)): os.remove(fh) except: pass raise finally: stopWatch.stop()
def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials) - proxy_server: 'http://my-proxy.site.com:8080' - proxy_user : '******' - proxy_pwd :'doe' * Autorisation mode: 'cas', 'basic', 'none' - auth_mode: 'cas' * User credentials for authentication 'cas' or 'basic' - user: '******' - pwd: 'doe' * Motu service URL - motu: 'http://atoll-dev.cls.fr:30080/mis-gateway-servlet/Motu' * Dataset identifier to download - product_id: 'dataset-duacs-global-nrt-madt-merged-h' * Service identifier to use for retrieving dataset - service_id: 'http://purl.org/myocean/ontology/service/database#yourduname' * Geographic extraction parameters - latitude_max : 10.0 - latitude_min : -10.0 - longitude_max: -0.333333333369 - longitude_min: 0.0 * Vertical extraction parameters - depth_max: 1000 - depth_min: 0 * Temporal extraction parameters, as a datetime instance or a string (format: '%Y-%m-%d %H:%M:%S') - date_max: 2010-04-25 12:05:36 - date_min: 2010-04-25 * Variable extraction - variable: ['variable1','variable2'] * The file name and the directory of the downloaded dataset - out_dir : '.' - out_name: 'dataset' * The block size used to perform download - block_size: 12001 * The socket timeout configuration - socket_timeout: 515 * The user agent to use when performing http requests - user_agent: 'motu-api-client' """ global log global init_time init_time = datetime.datetime.now() stopWatch = stop_watch.localThreadStopWatch() stopWatch.start() try: log = logging.getLogger("motu_api") # at first, we check given options are ok check_options(_options) # print some trace info about the options set log.log(utils_log.TRACE_LEVEL, '-' * 60) for option in dir(_options): if not option.startswith('_'): log.log(utils_log.TRACE_LEVEL, "%s=%s" % (option, getattr(_options, option))) log.log(utils_log.TRACE_LEVEL, '-' * 60) # start of url to invoke url_service = _options.motu # parameters of the invoked service url_params = build_params(_options) url_config = get_url_config(_options) # check if question mark is in the url questionMark = '?' if url_service.endswith(questionMark): questionMark = '' url = url_service + questionMark + url_params if _options.describe == True: _options.out_name = _options.out_name.replace('.nc', '.xml') # set-up the socket timeout if any if _options.socket_timeout != None: log.debug("Setting timeout %s" % _options.socket_timeout) socket.setdefaulttimeout(_options.socket_timeout) if _options.auth_mode == AUTHENTICATION_MODE_CAS: stopWatch.start('authentication') # perform authentication before acceding service download_url = utils_cas.authenticate_CAS_for_URL( url, _options.user, _options.pwd, **url_config) stopWatch.stop('authentication') else: # if none, we do nothing more, in basic, we let the url requester doing the job download_url = url # create a file for storing downloaded stream fh = os.path.join(_options.out_dir, _options.out_name) try: # Synchronous mode if _options.sync == True: dl_2_file(download_url, fh, _options.block_size, _options.describe, **url_config) log.info("Done") # Asynchronous mode else: stopWatch.start('wait_request') requestUrl = get_requestUrl(download_url, url_service, **url_config) # asynchronous mode status = 0 dwurl = "" while True: if _options.auth_mode == AUTHENTICATION_MODE_CAS: stopWatch.start('authentication') # perform authentication before acceding service requestUrlCas = utils_cas.authenticate_CAS_for_URL( requestUrl, _options.user, _options.pwd, **url_config) stopWatch.stop('authentication') else: # if none, we do nothing more, in basic, we let the url requester doing the job requestUrlCas = requestUrl m = utils_http.open_url(requestUrlCas, **url_config) motu_reply = m.read() dom = minidom.parseString(motu_reply) for node in dom.getElementsByTagName('statusModeResponse'): status = node.getAttribute('status') dwurl = node.getAttribute('remoteUri') # Check status if status == "0" or status == "3": # in progress/pending log.info( 'Product is not yet available (request in process)' ) time.sleep(10) else: # finished (error|success) break stopWatch.stop('wait_request') if status == "2": log.error(dwurl) if status == "1": log.info('The product is ready for download') if dwurl != "": dl_2_file(dwurl, fh, _options.block_size, _options.describe, **url_config) log.info("Done") else: log.error("Couldn't retrieve file") except: try: if (os.path.isfile(fh)): os.remove(fh) except: pass raise finally: stopWatch.stop()
def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials) - proxy_server: 'http://my-proxy.site.com:8080' - proxy_user : '******' - proxy_pwd :'doe' * Autorisation mode: 'cas', 'basic', 'none' - auth_mode: 'cas' * User credentials for authentication 'cas' or 'basic' - user: '******' - pwd: 'doe' * Motu service URL - motu: 'http://atoll-dev.cls.fr:30080/mis-gateway-servlet/Motu' * Dataset identifier to download - product_id: 'dataset-duacs-global-nrt-madt-merged-h' * Service identifier to use for retrieving dataset - service_id: 'http://purl.org/myocean/ontology/service/database#yourduname' * Geographic extraction parameters - latitude_max : 10.0 - latitude_min : -10.0 - longitude_max: -0.333333333369 - longitude_min: 0.0 * Vertical extraction parameters - depth_max: 1000 - depth_min: 0 * Temporal extraction parameters, as a datetime instance or a string (format: '%Y-%m-%d %H:%M:%S') - date_max: 2010-04-25 12:05:36 - date_min: 2010-04-25 * Variable extraction - variable: ['variable1','variable2'] * The file name and the directory of the downloaded dataset - out_dir : '.' - out_name: 'dataset' * The block size used to perform download - block_size: 12001 * The socket timeout configuration - socket_timeout: 515 * The user agent to use when performing http requests - user_agent: 'motu-api-client' """ global log stopWatch = stop_watch.localThreadStopWatch() stopWatch.start() try: log = logging.getLogger("motu_api") # at first, we check given options are ok check_options(_options) # print some trace info about the options set log.log(utils_log.TRACE_LEVEL, '-' * 60) for option in dir(_options): if not option.startswith('_'): log.log(utils_log.TRACE_LEVEL, "%s=%s" % (option, getattr(_options, option))) log.log(utils_log.TRACE_LEVEL, '-' * 60) # start of url to invoke url_service = _options.motu # parameters of the invoked service url_params = build_params(_options) url_config = get_url_config(_options) # check if question mark is in the url questionMark = '?' if url_service.endswith(questionMark): questionMark = '' url = url_service + questionMark + url_params # set-up the socket timeout if any if _options.socket_timeout != None: log.debug("Setting timeout %s" % _options.socket_timeout) socket.setdefaulttimeout(_options.socket_timeout) if _options.auth_mode == AUTHENTICATION_MODE_CAS: stopWatch.start('authentication') # perform authentication before acceding service download_url = utils_cas.authenticate_CAS_for_URL( url, _options.user, _options.pwd, **url_config) stopWatch.stop('authentication') else: # if none, we do nothing more, in basic, we let the url requester doing the job download_url = url # create a file for storing downloaded stream fh = os.path.join(_options.out_dir, _options.out_name) try: dl_2_file(download_url, fh, _options.block_size, **url_config) log.info("Done") except: try: if (os.path.isfile(fh)): os.remove(fh) except: pass raise finally: stopWatch.stop()