def check_options(_options):
"""function that checks the given options for coherency."""
# Check Mandatory Options
if ((_options.auth_mode != AUTHENTICATION_MODE_NONE and
_options.auth_mode != AUTHENTICATION_MODE_BASIC and
_options.auth_mode != AUTHENTICATION_MODE_CAS)):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.invalid'
] % (_options.auth_mode,
'auth-mode',
[AUTHENTICATION_MODE_NONE,
AUTHENTICATION_MODE_BASIC,
AUTHENTICATION_MODE_CAS])
)
# if authentication mode is set we check both user & password presence
if ((_options.user is None and
_options.auth_mode != AUTHENTICATION_MODE_NONE)):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory.user'
] % ('user', _options.auth_mode))
# check that if a user is set, a password should be set also
if ((_options.pwd is None and
_options.user is not None)):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory.password'
] % ('pwd', _options.user))
# check that if a user is set, an authentication mode should also be set
if ((_options.user is not None and
_options.auth_mode == AUTHENTICATION_MODE_NONE)):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory.mode'
] % (AUTHENTICATION_MODE_NONE, 'auth-mode', _options.user))
# those following parameters are required
if _options.motu is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory'
] % 'motu')
if _options.service_id is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory'
] % 'service-id')
if _options.product_id is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory'
] % 'product-id')
if _options.out_dir is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory'
] % 'out-dir')
out_dir = _options.out_dir
# check directory existence
if not os.path.exists(out_dir):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.outdir-notexist'
] % out_dir)
# check whether directory is writable or not
if not os.access(out_dir, os.W_OK):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.outdir-notwritable'
] % out_dir)
if _options.out_name is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.mandatory'
] % 'out-name')
# Check PROXY Options
_options.proxy = False
if (((_options.proxy_server is not None) and
(len(_options.proxy_server) != 0))):
_options.proxy = True
# check that proxy server is a valid url
url = _options.proxy_server
p = re.compile('^(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)'
'(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?')
m = p.match(url)
if not m:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.not-url'
] % ('proxy-server', url))
# check that if proxy-user is defined then proxy-pwd shall be also,
# and reciprocally.
if (((_options.proxy_user is not None) !=
(_options.proxy_pwd is not None))):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.linked'
] % ('proxy-user', 'proxy-name'))
# Check VERTICAL Options
_options.extraction_vertical = False
if _options.depth_min is not None or _options.depth_max is not None:
_options.extraction_vertical = True
# Check TEMPORAL Options
_options.extraction_temporal = False
if _options.date_min is not None or _options.date_max is not None:
_options.extraction_temporal = True
# Check GEOGRAPHIC Options
_options.extraction_geographic = False
if ((_options.latitude_min is not None or
_options.latitude_max is not None or
_options.longitude_min is not None or
_options.longitude_max is not None)):
_options.extraction_geographic = True
if _options.latitude_min is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.geographic-box'
] % 'latitude_min')
if _options.latitude_max is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.geographic-box'
] % 'latitude_max')
if _options.longitude_min is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.geographic-box'
] % 'longitude_min')
if _options.longitude_max is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.geographic-box'
] % 'longitude_max')
tempvalue = float(_options.latitude_min)
if tempvalue < -90 or tempvalue > 90:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.out-of-range'
] % ('latitude_min', str(tempvalue)))
tempvalue = float(_options.latitude_max)
if tempvalue < -90 or tempvalue > 90:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.out-of-range'
] % ('latitude_max', str(tempvalue)))
tempvalue = float(_options.longitude_min)
if tempvalue < -180 or tempvalue > 180:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.out-of-range'
] % ('logitude_min', str(tempvalue)))
tempvalue = float(_options.longitude_max)
if tempvalue < -180 or tempvalue > 180:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.option.out-of-range'
] % ('longitude_max', str(tempvalue)))
def dl_2_file(dl_url, fh, block_size=65535, describe='None', **options):
""" Download the file with the main url (of Motu) file.
Motu can return an error message in the response stream without setting an
appropriate http error code. So, in that case, the content-type response is
checked, and if it is text/plain, we consider this as an error.
dl_url: the complete download url of Motu
fh: file handler to use to write the downstream
"""
stopWatch = stop_watch.localThreadStopWatch()
start_time = datetime.datetime.now()
log.info("Downloading file (this can take a while)...")
# download file
temp = open(fh, 'w+b')
try:
stopWatch.start('processing')
m = utils_http.open_url(dl_url, **options)
try:
# check the real url (after potential redirection) is not a CAS
# Url scheme
match = re.search(utils_cas.CAS_URL_PATTERN, m.url)
if match is not None:
service, _, _ = dl_url.partition('?')
redirection, _, _ = m.url.partition('?')
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.authentication.redirected'
] % (service, redirection))
# check that content type is not text/plain
headers = m.info()
if "Content-Type" in headers:
if len(headers['Content-Type']) > 0:
if ((headers['Content-Type'].startswith('text') or
headers['Content-Type'].find('html') != -1)):
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.motu.error'
] % m.read())
log.info('File type: %s' % headers['Content-Type'])
# check if a content length (size of the file) has been send
if "Content-Length" in headers:
try:
# it should be an integer
size = int(headers["Content-Length"])
log.info('File size: %s (%i B)' % (
utils_unit.convert_bytes(size), size))
except Exception as e:
size = -1
log.warn(
'File size is not an integer: %s' % headers[
"Content-Length"])
else:
size = -1
log.warn('File size: %s' % 'unknown')
processing_time = datetime.datetime.now()
stopWatch.stop('processing')
stopWatch.start('downloading')
# performs the download
log.info('Downloading file %s' % os.path.abspath(fh))
def progress_function(sizeRead):
percent = sizeRead*100./size
log.info("- %s (%.1f%%)",
utils_unit.convert_bytes(size).rjust(8),
percent)
def none_function(sizeRead):
percent = 100
log.info("- %s (%.1f%%)",
utils_unit.convert_bytes(size).rjust(8),
percent)
read = utils_stream.copy(m,
temp,
progress_function if size != -1
else none_function,
block_size)
end_time = datetime.datetime.now()
stopWatch.stop('downloading')
log.info("Processing time : %s",
str(processing_time - start_time))
log.info("Downloading time : %s", str(end_time - processing_time))
log.info("Total time : %s", str(end_time - start_time))
log.info("Download rate : %s/s", utils_unit.convert_bytes(
(read / total_milliseconds(end_time - start_time)) * 10**3))
finally:
m.close()
finally:
temp.flush()
temp.close()
# raise exception if actual size does not match content-length header
if size >= 0 and read < size:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.download.too-short'
] % (read, size))
def authenticate_CAS_for_URL(url, user, pwd, **url_config):
"""Performs a CAS authentication for the given URL service and returns
the service url with the obtained credential.
The following algorithm is done:
1) A connection is opened on the given URL
2) We check that the response is an HTTP redirection
3) Redirected URL contains the CAS address
4) We ask for a ticket for the given user and password
5) We ask for a service ticket for the given service
6) Then we return a new url with the ticket attached
url: the url of the service to invoke
user: the username
pwd: the password"""
log = logging.getLogger("utils_cas:authenticate_CAS_for_URL")
server, _, _ = url.partition('?')
log.info('Authenticating user %s for service %s', user, server)
connexion = utils_http.open_url(url, **url_config)
# connexion response code must be a redirection, else, there's an error
# (user can't be already connected since no cookie or ticket was sent)
if connexion.url == url:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.authentication.not-redirected'
] % server)
# find the cas url from the redirected url
redirected_url = connexion.url
m = re.search(CAS_URL_PATTERN, redirected_url)
if m is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.authentication.unfound-url'
] % redirected_url)
url_cas = m.group(1) + '/v1/tickets'
opts = utils_http.encode(utils_collection.ListMultimap(username=user,
password=pwd))
utils_log.log_url(log, "login user into CAS:\t", url_cas+'?'+opts)
url_config['data'] = opts
connexion = utils_http.open_url(url_cas, **url_config)
fp = utils_html.FounderParser()
for line in connexion:
log.log(utils_log.TRACE_LEVEL,
'utils_html.FounderParser() line: %s',
line)
fp.feed(line)
tgt = fp.action_[fp.action_.rfind('/') + 1:]
log.log(utils_log.TRACE_LEVEL,
'TGT: %s',
tgt)
# WARNING : don't use 'fp.action_' as url : it seems protocol is always
# http never https use 'url_cas', extract TGT from 'fp.action_' , then
# construct url_ticket.
# url_ticket = fp.action_
url_ticket = url_cas + '/' + tgt
if url_ticket is None:
raise Exception(
utils_messages.get_external_messages()[
'motu-client.exception.authentication.tgt'
])
utils_log.log_url(log, "found url ticket:\t", url_ticket)
opts = utils_http.encode(
utils_collection.ListMultimap(service=urllib.quote_plus(url)))
utils_log.log_url(log,
'Granting user for service\t',
url_ticket + '?' + opts)
url_config['data'] = opts
ticket = utils_http.open_url(url_ticket, **url_config).readline()
utils_log.log_url(log, "found service ticket:\t", ticket)
# we append the download url with the ticket and return the result
service_url = url + '&ticket=' + ticket
utils_log.log_url(log, "service url is:\t", service_url)
return service_url
