def dl_2_file(dl_url, fh, block_size=65535, describe=None, **options): """ Download the file with the main url (of Motu) file. Motu can return an error message in the response stream without setting an appropriate http error code. So, in that case, the content-type response is checked, and if it is text/plain, we consider this as an error. dl_url: the complete download url of Motu fh: file handler to use to write the downstream""" stopWatch = stop_watch.localThreadStopWatch() start_time = datetime.datetime.now() log.info("Downloading file (this can take a while)...") # download file temp = open(fh, 'w+b') try: stopWatch.start('processing') m = utils_http.open_url(dl_url, **options) try: # check the real url (after potential redirection) is not a CAS Url scheme match = re.search(utils_cas.CAS_URL_PATTERN, m.url) if match is not None: service, _, _ = dl_url.partition('?') redirection, _, _ = m.url.partition('?') raise Exception( utils_messages.get_external_messages() ['motu-client.exception.authentication.redirected'] % (service, redirection)) # check that content type is not text/plain headers = m.info() if "Content-Type" in headers: if len(headers['Content-Type']) > 0: if not describe: if headers['Content-Type'].startswith( 'text' ) or headers['Content-Type'].find('html') != -1: raise Exception( utils_messages.get_external_messages() ['motu-client.exception.motu.error'] % m.read()) log.info('File type: %s' % headers['Content-Type']) # check if a content length (size of the file) has been send if "Content-Length" in headers: try: # it should be an integer size = int(headers["Content-Length"]) log.info('File size: %s (%i B)' % (utils_unit.convert_bytes(size), size)) except Exception, e: size = -1 log.warn('File size is not an integer: %s' % headers["Content-Length"]) else: size = -1 log.warn('File size: %s' % 'unknown')
def dl_2_file(dl_url, fh, block_size = 65535, describe = 'None', **options): """ Download the file with the main url (of Motu) file. Motu can return an error message in the response stream without setting an appropriate http error code. So, in that case, the content-type response is checked, and if it is text/plain, we consider this as an error. dl_url: the complete download url of Motu fh: file handler to use to write the downstream""" stopWatch = stop_watch.localThreadStopWatch() start_time = datetime.datetime.now() log.info( "Downloading file (this can take a while)..." ) # download file temp = open(fh, 'w+b') try: stopWatch.start('processing') m = utils_http.open_url(dl_url, **options) try: # check the real url (after potential redirection) is not a CAS Url scheme match = re.search(utils_cas.CAS_URL_PATTERN, m.url) if match is not None: service, _, _ = dl_url.partition('?') redirection, _, _ = m.url.partition('?') raise Exception(utils_messages.get_external_messages()['motu-client.exception.authentication.redirected'] % (service, redirection) ) # check that content type is not text/plain headers = m.info() if "Content-Type" in headers: if len(headers['Content-Type']) > 0: if headers['Content-Type'].startswith('text') or headers['Content-Type'].find('html') != -1: raise Exception( utils_messages.get_external_messages()['motu-client.exception.motu.error'] % m.read() ) log.info( 'File type: %s' % headers['Content-Type'] ) # check if a content length (size of the file) has been send if "Content-Length" in headers: try: # it should be an integer size = int(headers["Content-Length"]) log.info( 'File size: %s (%i B)' % ( utils_unit.convert_bytes(size), size ) ) except Exception, e: size = -1 log.warn( 'File size is not an integer: %s' % headers["Content-Length"] ) else: size = -1 log.warn( 'File size: %s' % 'unknown' )
def check_options(_options): """function that checks the given options for coherency.""" # Check Mandatory Options if (_options.auth_mode != AUTHENTICATION_MODE_NONE and _options.auth_mode != AUTHENTICATION_MODE_BASIC and _options.auth_mode != AUTHENTICATION_MODE_CAS): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.invalid'] % (_options.auth_mode, 'auth-mode', [ AUTHENTICATION_MODE_NONE, AUTHENTICATION_MODE_BASIC, AUTHENTICATION_MODE_CAS ])) # if authentication mode is set we check both user & password presence if (_options.user == None and _options.auth_mode != AUTHENTICATION_MODE_NONE): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory.user'] % ('user', _options.auth_mode)) # check that if a user is set, a password should be set also if (_options.pwd == None and _options.user != None): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory.password'] % ('pwd', _options.user)) #check that if a user is set, an authentication mode should also be set if (_options.user != None and _options.auth_mode == AUTHENTICATION_MODE_NONE): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory.mode'] % (AUTHENTICATION_MODE_NONE, 'auth-mode', _options.user)) # those following parameters are required if _options.motu == None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory'] % 'motu') if _options.service_id == None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory'] % 'service-id') if _options.product_id == None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory'] % 'product-id') if _options.out_dir == None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory'] % 'out-dir') out_dir = _options.out_dir # check directory existence if not os.path.exists(out_dir): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.outdir-notexist'] % out_dir) # check whether directory is writable or not if not os.access(out_dir, os.W_OK): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.outdir-notwritable'] % out_dir) if _options.out_name == None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.mandatory'] % 'out-name') # Check PROXY Options _options.proxy = False if (_options.proxy_server != None) and (len(_options.proxy_server) != 0): _options.proxy = True # check that proxy server is a valid url url = _options.proxy_server p = re.compile( '^(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?' ) m = p.match(url) if not m: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.not-url'] % ('proxy-server', url)) # check that if proxy-user is defined then proxy-pwd shall be also, and reciprocally. if (_options.proxy_user != None) != (_options.proxy_pwd != None): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.linked'] % ('proxy-user', 'proxy-name')) # Check VERTICAL Options _options.extraction_vertical = False if _options.depth_min != None or _options.depth_max != None: _options.extraction_vertical = True # Check TEMPORAL Options _options.extraction_temporal = False if _options.date_min != None or _options.date_max != None: _options.extraction_temporal = True # Check GEOGRAPHIC Options _options.extraction_geographic = False if _options.latitude_min != None or _options.latitude_max != None or _options.longitude_min != None or _options.longitude_max != None: _options.extraction_geographic = True if (_options.latitude_min == None): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.geographic-box'] % 'latitude_min') if (_options.latitude_max == None): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.geographic-box'] % 'latitude_max') if (_options.longitude_min == None): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.geographic-box'] % 'longitude_min') if (_options.longitude_max == None): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.geographic-box'] % 'longitude_max') tempvalue = float(_options.latitude_min) if tempvalue < -90 or tempvalue > 90: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.out-of-range'] % ('latitude_min', str(tempvalue))) tempvalue = float(_options.latitude_max) if tempvalue < -90 or tempvalue > 90: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.out-of-range'] % ('latitude_max', str(tempvalue))) tempvalue = float(_options.longitude_min) tempvalue = normalize_longitude(tempvalue) if tempvalue < -180 or tempvalue > 180: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.out-of-range'] % ('logitude_min', str(tempvalue))) tempvalue = float(_options.longitude_max) tempvalue = normalize_longitude(tempvalue) if tempvalue < -180 or tempvalue > 180: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.option.out-of-range'] % ('longitude_max', str(tempvalue)))
log.info("Downloading time : %s", str(end_time - processing_time)) log.info("Total time : %s", str(end_time - init_time)) log.info( "Download rate : %s/s", utils_unit.convert_bytes( (read / total_milliseconds(end_time - start_time)) * 10**3)) finally: m.close() finally: temp.flush() temp.close() # raise exception if actual size does not match content-length header if size >= 0 and read < size: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.download.too-short'] % (read, size)) def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials) - proxy_server: 'http://my-proxy.site.com:8080' - proxy_user : '******' - proxy_pwd :'doe' * Autorisation mode: 'cas', 'basic', 'none' - auth_mode: 'cas'
def authenticate_CAS_for_URL(url, user, pwd, **url_config): """Performs a CAS authentication for the given URL service and returns the service url with the obtained credential. The following algorithm is done: 1) A connection is opened on the given URL 2) We check that the response is an HTTP redirection 3) Redirected URL contains the CAS address 4) We ask for a ticket for the given user and password 5) We ask for a service ticket for the given service 6) Then we return a new url with the ticket attached url: the url of the service to invoke user: the username pwd: the password""" log = logging.getLogger("utils_cas:authenticate_CAS_for_URL") server, sep, options = url.partition('?') log.info('Authenticating user %s for service %s' % (user, server)) connexion = utils_http.open_url(url, **url_config) # connexion response code must be a redirection, else, there's an error (user can't be already connected since no cookie or ticket was sent) if connexion.url == url: raise Exception( utils_messages.get_external_messages() ['motu-client.exception.authentication.not-redirected'] % server) # find the cas url from the redirected url redirected_url = connexion.url p = parse_qs(urlparse(connexion.url).query, keep_blank_values=False) redirectServiceUrl = p['service'][0] m = re.search(CAS_URL_PATTERN, redirected_url) if m is None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.authentication.unfound-url'] % redirected_url) url_cas = m.group(1) + '/v1/tickets' opts = utils_http.encode( utils_collection.ListMultimap(username=urllib.quote(user), password=urllib.quote(pwd))) utils_log.log_url(log, "login user into CAS:\t", url_cas + '?' + opts) url_config['data'] = opts connexion = utils_http.open_url(url_cas, **url_config) fp = utils_html.FounderParser() for line in connexion: log.log(utils_log.TRACE_LEVEL, 'utils_html.FounderParser() line: %s', line) fp.feed(line) tgt = fp.action_[fp.action_.rfind('/') + 1:] log.log(utils_log.TRACE_LEVEL, 'TGT: %s', tgt) # WARNING : don't use 'fp.action_' as url : it seems protocol is always http never https # use 'url_cas', extract TGT from 'fp.action_' , then construct url_ticket. # url_ticket = fp.action_ url_ticket = url_cas + '/' + tgt if url_ticket is None: raise Exception(utils_messages.get_external_messages() ['motu-client.exception.authentication.tgt']) utils_log.log_url(log, "found url ticket:\t", url_ticket) opts = utils_http.encode( utils_collection.ListMultimap( service=urllib.quote_plus(redirectServiceUrl))) utils_log.log_url(log, 'Granting user for service\t', url_ticket + '?' + opts) url_config['data'] = opts ticket = utils_http.open_url(url_ticket, **url_config).readline() utils_log.log_url(log, "found service ticket:\t", ticket) # we append the download url with the ticket and return the result service_url = redirectServiceUrl + '&ticket=' + ticket utils_log.log_url(log, "service url is:\t", service_url) return service_url
def dl_2_file(dl_url, fh, block_size=65535, isADownloadRequest=None, **options): """ Download the file with the main url (of Motu) file. Motu can return an error message in the response stream without setting an appropriate http error code. So, in that case, the content-type response is checked, and if it is text/plain, we consider this as an error. dl_url: the complete download url of Motu fh: file handler to use to write the downstream""" stopWatch = stop_watch.localThreadStopWatch() start_time = datetime.datetime.now() log.info("Downloading file (this can take a while)...") # download file temp = None if not fh.startswith("console"): temp = open(fh, 'w+b') try: stopWatch.start('processing') m = utils_http.open_url(dl_url, **options) try: # check the real url (after potential redirection) is not a CAS Url scheme match = re.search(utils_cas.CAS_URL_PATTERN, m.url) if match is not None: service, _, _ = dl_url.partition('?') redirection, _, _ = m.url.partition('?') raise Exception( utils_messages.get_external_messages() ['motu-client.exception.authentication.redirected'] % (service, redirection)) # check that content type is not text/plain headers = m.info() if "Content-Type" in headers and len( headers['Content-Type']) > 0 and isADownloadRequest and ( headers['Content-Type'].startswith('text') or headers['Content-Type'].find('html') != -1): raise Exception(utils_messages.get_external_messages() ['motu-client.exception.motu.error'] % m.read()) log.info('File type: %s' % headers['Content-Type']) # check if a content length (size of the file) has been send size = -1 if "Content-Length" in headers: try: # it should be an integer size = int(headers["Content-Length"]) log.info('File size: %s (%i B)' % (utils_unit.convert_bytes(size), size)) except Exception, e: size = -1 log.warn('File size is not an integer: %s' % headers["Content-Length"]) elif temp is not None: log.warn('File size: %s' % 'unknown') processing_time = datetime.datetime.now() stopWatch.stop('processing') stopWatch.start('downloading') # performs the download log.info('Downloading file %s' % os.path.abspath(fh)) def progress_function(sizeRead): percent = sizeRead * 100. / size log.info("- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent) td = datetime.datetime.now() - start_time def none_function(sizeRead): percent = 100 log.info("- %s (%.1f%%)", utils_unit.convert_bytes(size).rjust(8), percent) td = datetime.datetime.now() - start_time if temp is not None: read = utils_stream.copy( m, temp, progress_function if size != -1 else none_function, block_size) else: if isADownloadRequest: #Console mode, only display the NC file URL on stdout read = len(m.url) print(m.url) else: import cStringIO output = cStringIO.StringIO() utils_stream.copy( m, output, progress_function if size != -1 else none_function, block_size) read = len(output.getvalue()) print(output.getvalue()) end_time = datetime.datetime.now() stopWatch.stop('downloading') log.info("Processing time : %s", str(processing_time - init_time)) log.info("Downloading time : %s", str(end_time - processing_time)) log.info("Total time : %s", str(end_time - init_time)) log.info( "Download rate : %s/s", utils_unit.convert_bytes( (read / total_milliseconds(end_time - start_time)) * 10**3))
log.info("Total time : %s", str(end_time - start_time)) log.info( "Download rate : %s/s", utils_unit.convert_bytes( (read / total_milliseconds(end_time - start_time)) * 10**3)) finally: m.close() finally: temp.flush() temp.close() # raise exception if actual size does not match content-length header if size >= 0 and read < size: raise ContentTooShortError( utils_messages.get_external_messages() ['motu-client.exception.download.too-short'] % (read, size), result) def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials) - proxy_server: 'http://my-proxy.site.com:8080' - proxy_user : '******' - proxy_pwd :'doe' * Autorisation mode: 'cas', 'basic', 'none' - auth_mode: 'cas'
def check_options(_options): """function that checks the given options for coherency.""" # Check Mandatory Options if (_options.auth_mode != AUTHENTICATION_MODE_NONE and _options.auth_mode != AUTHENTICATION_MODE_BASIC and _options.auth_mode != AUTHENTICATION_MODE_CAS): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.invalid'] % ( _options.auth_mode, 'auth-mode', [AUTHENTICATION_MODE_NONE, AUTHENTICATION_MODE_BASIC, AUTHENTICATION_MODE_CAS]) ) # if authentication mode is set we check both user & password presence if (_options.user == None and _options.auth_mode != AUTHENTICATION_MODE_NONE): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory.user'] % ('user',_options.auth_mode)) # check that if a user is set, a password should be set also if (_options.pwd == None and _options.user != None): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory.password'] % ( 'pwd', _options.user ) ) #check that if a user is set, an authentication mode should also be set if (_options.user != None and _options.auth_mode == AUTHENTICATION_MODE_NONE): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory.mode'] % ( AUTHENTICATION_MODE_NONE, 'auth-mode', _options.user ) ) # those following parameters are required if _options.motu == None : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory'] % 'motu') if _options.service_id == None : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory'] % 'service-id') if _options.product_id == None : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory'] % 'product-id') if _options.out_dir == None : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory'] % 'out-dir') out_dir = _options.out_dir # check directory existence if not os.path.exists(out_dir): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.outdir-notexist'] % out_dir) # check whether directory is writable or not if not os.access(out_dir, os.W_OK): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.outdir-notwritable'] % out_dir) if _options.out_name == None : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.mandatory'] % 'out-name') # Check PROXY Options _options.proxy = False if (_options.proxy_server != None) and (len(_options.proxy_server) != 0): _options.proxy = True # check that proxy server is a valid url url = _options.proxy_server p = re.compile('^(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?') m = p.match(url) if not m : raise Exception( utils_messages.get_external_messages()['motu-client.exception.option.not-url'] % ( 'proxy-server', url ) ) # check that if proxy-user is defined then proxy-pwd shall be also, and reciprocally. if (_options.proxy_user != None) != ( _options.proxy_pwd != None ) : raise Exception( utils_messages.get_external_messages()['motu-client.exception.option.linked'] % ('proxy-user', 'proxy-name') ) # Check VERTICAL Options _options.extraction_vertical = False if _options.depth_min != None or _options.depth_max != None : _options.extraction_vertical = True # Check TEMPORAL Options _options.extraction_temporal = False if _options.date_min != None or _options.date_max != None : _options.extraction_temporal = True # Check GEOGRAPHIC Options _options.extraction_geographic = False if _options.latitude_min != None or _options.latitude_max != None or _options.longitude_min != None or _options.longitude_max != None : _options.extraction_geographic = True if( _options.latitude_min == None ): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.geographic-box'] % 'latitude_min' ) if( _options.latitude_max == None ): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.geographic-box'] % 'latitude_max' ) if( _options.longitude_min == None ): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.geographic-box'] % 'longitude_min' ) if( _options.longitude_max == None ): raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.geographic-box'] % 'longitude_max' ) tempvalue = float(_options.latitude_min) if tempvalue < -90 or tempvalue > 90 : raise Exception( utils_messages.get_external_messages()['motu-client.exception.option.out-of-range'] % ( 'latitude_min', str(tempvalue)) ) tempvalue = float(_options.latitude_max) if tempvalue < -90 or tempvalue > 90 : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.out-of-range'] % ( 'latitude_max', str(tempvalue))) tempvalue = float(_options.longitude_min) if tempvalue < -180 or tempvalue > 180 : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.out-of-range'] % ( 'logitude_min', str(tempvalue))) tempvalue = float(_options.longitude_max) if tempvalue < -180 or tempvalue > 180 : raise Exception(utils_messages.get_external_messages()['motu-client.exception.option.out-of-range'] % ( 'longitude_max', str(tempvalue)))
end_time = datetime.datetime.now() stopWatch.stop('downloading') log.info( "Processing time : %s", str(processing_time - start_time) ) log.info( "Downloading time : %s", str(end_time - processing_time) ) log.info( "Total time : %s", str(end_time - start_time) ) log.info( "Download rate : %s/s", utils_unit.convert_bytes((read / total_milliseconds(end_time - start_time)) * 10**3) ) finally: m.close() finally: temp.flush() temp.close() # raise exception if actual size does not match content-length header if size >= 0 and read < size: raise ContentTooShortError( utils_messages.get_external_messages()['motu-client.exception.download.too-short'] % (read, size), result) def execute_request(_options): """ the main function that submit a request to motu. Available options are: * Proxy configuration (with eventually user credentials) - proxy_server: 'http://my-proxy.site.com:8080' - proxy_user : '******' - proxy_pwd :'doe' * Autorisation mode: 'cas', 'basic', 'none' - auth_mode: 'cas' * User credentials for authentication 'cas' or 'basic' - user: '******'
def authenticate_CAS_for_URL(url, user, pwd, **url_config): """Performs a CAS authentication for the given URL service and returns the service url with the obtained credential. The following algorithm is done: 1) A connection is opened on the given URL 2) We check that the response is an HTTP redirection 3) Redirected URL contains the CAS address 4) We ask for a ticket for the given user and password 5) We ask for a service ticket for the given service 6) Then we return a new url with the ticket attached url: the url of the service to invoke user: the username pwd: the password""" log = logging.getLogger("utils_cas:authenticate_CAS_for_URL") server, sep, options = url.partition( '?' ) log.info( 'Authenticating user %s for service %s' % (user,server) ) connexion = utils_http.open_url(url,**url_config) # connexion response code must be a redirection, else, there's an error (user can't be already connected since no cookie or ticket was sent) if connexion.url == url: raise Exception(utils_messages.get_external_messages()['motu-client.exception.authentication.not-redirected'] % server ) # find the cas url from the redirected url redirected_url = connexion.url m = re.search(CAS_URL_PATTERN, redirected_url) if m is None: raise Exception(utils_messages.get_external_messages()['motu-client.exception.authentication.unfound-url'] % redirected_url) url_cas = m.group(1) + '/v1/tickets' opts = utils_http.encode(utils_collection.ListMultimap(username = user, password = pwd)) utils_log.log_url( log, "login user into CAS:\t", url_cas+'?'+opts ) url_config['data']=opts connexion = utils_http.open_url(url_cas, **url_config) fp = utils_html.FounderParser() for line in connexion: log.log( utils_log.TRACE_LEVEL, 'utils_html.FounderParser() line: %s', line ) fp.feed(line) tgt = fp.action_[fp.action_.rfind('/') + 1:] log.log( utils_log.TRACE_LEVEL, 'TGT: %s', tgt ) # WARNING : don't use 'fp.action_' as url : it seems protocol is always http never https # use 'url_cas', extract TGT from 'fp.action_' , then construct url_ticket. # url_ticket = fp.action_ url_ticket = url_cas + '/' + tgt if url_ticket is None: raise Exception(utils_messages.get_external_messages()['motu-client.exception.authentication.tgt']) utils_log.log_url( log, "found url ticket:\t",url_ticket) opts = utils_http.encode(utils_collection.ListMultimap(service = urllib.quote_plus(url))) utils_log.log_url( log, 'Granting user for service\t', url_ticket +'?'+opts ) url_config['data']=opts ticket = utils_http.open_url(url_ticket, **url_config).readline() utils_log.log_url( log, "found service ticket:\t", ticket) # we append the download url with the ticket and return the result service_url = url + '&ticket=' + ticket utils_log.log_url( log, "service url is:\t",service_url) return service_url