def file_handler(complexinput, datain):
"""<wps:Reference /> handler.
Used when href is a file url."""
# check if file url is allowed
_validate_file_input(href=datain.get('href'))
# save the file reference input in workdir
tmp_file = _build_input_file_name(
href=datain.get('href'),
workdir=complexinput.workdir,
extension=_extension(complexinput))
try:
inpt_file = urlparse(datain.get('href')).path
inpt_file = os.path.abspath(inpt_file)
os.symlink(inpt_file, tmp_file)
LOGGER.debug("Linked input file %s to %s.", inpt_file,
tmp_file)
except Exception:
# TODO: handle os.symlink on windows
# raise NoApplicableCode("Could not link file reference: %s" % e)
LOGGER.warn("Could not link file reference")
shutil.copy2(inpt_file, tmp_file)
complexinput.file = tmp_file
complexinput.url = datain.get('href')
complexinput.as_reference = True
def test_url_handler(self):
wfsResource = 'http://demo.mapserver.org/cgi-bin/wfs?' \
'service=WFS&version=1.1.0&' \
'request=GetFeature&' \
'typename=continents&maxfeatures=2'
self.complex_out.url = wfsResource
self.complex_out.storage = FileStorageBuilder().build()
url = self.complex_out.get_url()
self.assertEqual('file', urlparse(url).scheme)
def _test_outout(self, source_type, suffix=''):
"""Test all outputs"""
self.assertEqual(source_type, self.iohandler.source_type,
'Source type properly set')
self.assertEqual(self._value, self.iohandler.data, 'Data obtained')
if self.iohandler.source_type == SOURCE_TYPE.URL:
self.assertEqual('http', urlparse(self.iohandler.url).scheme)
else:
self.assertEqual('file', urlparse(self.iohandler.url).scheme)
if self.iohandler.source_type == SOURCE_TYPE.STREAM:
source = StringIO(text_type(self._value))
self.iohandler.stream = source
file_path = self.iohandler.file
self.assertTrue(file_path.endswith(suffix))
file_handler = open(file_path)
self.assertEqual(self._value, file_handler.read(), 'File obtained')
file_handler.close()
if self.iohandler.source_type == SOURCE_TYPE.STREAM:
source = StringIO(text_type(self._value))
self.iohandler.stream = source
stream_val = self.iohandler.stream.read()
self.iohandler.stream.close()
if PY2 and isinstance(stream_val, str):
self.assertEqual(self._value, stream_val.decode('utf-8'),
'Stream obtained')
elif not PY2 and isinstance(stream_val, bytes):
self.assertEqual(self._value, stream_val.decode('utf-8'),
'Stream obtained')
else:
self.assertEqual(self._value, stream_val,
'Stream obtained')
if self.iohandler.source_type == SOURCE_TYPE.STREAM:
source = StringIO(text_type(self._value))
self.iohandler.stream = source
def _test_outout(self, source_type, suffix=''):
"""Test all outputs"""
self.assertEqual(source_type, self.iohandler.source_type,
'Source type properly set')
self.assertEqual(self._value, self.iohandler.data, 'Data obtained')
if self.iohandler.source_type == SOURCE_TYPE.URL:
self.assertEqual('http', urlparse(self.iohandler.url).scheme)
else:
self.assertEqual('file', urlparse(self.iohandler.url).scheme)
if self.iohandler.source_type == SOURCE_TYPE.STREAM:
source = StringIO(text_type(self._value))
self.iohandler.stream = source
file_path = self.iohandler.file
self.assertTrue(file_path.endswith(suffix))
file_handler = open(file_path)
self.assertEqual(self._value, file_handler.read(), 'File obtained')
file_handler.close()
if self.iohandler.source_type == SOURCE_TYPE.STREAM:
source = StringIO(text_type(self._value))
self.iohandler.stream = source
stream_val = self.iohandler.stream.read()
self.iohandler.stream.close()
if PY2 and isinstance(stream_val, str):
self.assertEqual(self._value, stream_val.decode('utf-8'),
'Stream obtained')
elif not PY2 and isinstance(stream_val, bytes):
self.assertEqual(self._value, stream_val.decode('utf-8'),
'Stream obtained')
else:
self.assertEqual(self._value, stream_val,
'Stream obtained')
if self.iohandler.source_type == SOURCE_TYPE.STREAM:
source = StringIO(text_type(self._value))
self.iohandler.stream = source
def test_url_handler(self):
wfsResource = 'http://demo.mapserver.org/cgi-bin/wfs?' \
'service=WFS&version=1.1.0&' \
'request=GetFeature&' \
'typename=continents&maxfeatures=2'
self.complex_out.url = wfsResource
storage = FileStorage()
self.complex_out.storage = storage
url = self.complex_out.get_url()
self.assertEqual('file', urlparse(url).scheme)
def test_file_handler(self):
self.complex_out.file = self.test_fn
self.assertEqual(self.complex_out.data, self.data)
if PY2:
self.assertEqual(self.complex_out.stream.read(), self.data)
else:
with self.complex_out.stream as s:
self.assertEqual(s.read(), bytes(self.data, encoding='utf8'))
with open(urlparse(self.complex_out.url).path) as f:
self.assertEqual(f.read(), self.data)
def test_file_handler(self):
self.complex_out.file = self.test_fn
self.assertEqual(self.complex_out.data, self.data)
if PY2:
self.assertEqual(self.complex_out.stream.read(), self.data)
else:
with self.complex_out.stream as s:
self.assertEqual(s.read(), bytes(self.data, encoding='utf8'))
with open(urlparse(self.complex_out.url).path) as f:
self.assertEqual(f.read(), self.data)
def convert_anyURI(inpt):
"""Return value of input
:rtype: url components
"""
inpt = convert_string(inpt)
components = urlparse(inpt)
if (components[0] and components[1]) or components[0] == 'file':
return components
else:
raise InvalidParameterValue(
'The value "{}" does not seem to be of type anyURI'.format(inpt))
def get_host():
url = configuration.get_config_value('server', 'url')
url = url or 'http://localhost:5000/wps'
LOGGER.warn("starting WPS service on %s", url)
parsed_url = urlparse(url)
if ':' in parsed_url.netloc:
host, port = parsed_url.netloc.split(':')
port = int(port)
else:
host = parsed_url.netloc
port = 80
return host, port
def _build_input_file_name(href, workdir, extension=None):
href = href or ''
url_path = urlparse(href).path or ''
file_name = os.path.basename(url_path).strip() or 'input'
(prefix, suffix) = os.path.splitext(file_name)
suffix = suffix or extension or ''
if prefix and suffix:
file_name = prefix + suffix
input_file_name = os.path.join(workdir, file_name)
# build tempfile in case of duplicates
if os.path.exists(input_file_name):
input_file_name = tempfile.mkstemp(
suffix=suffix, prefix=prefix + '_',
dir=workdir)[1]
return input_file_name
def _build_input_file_name(href, workdir, extension=None):
href = href or ''
url_path = urlparse(href).path or ''
file_name = os.path.basename(url_path).strip() or 'input'
(prefix, suffix) = os.path.splitext(file_name)
suffix = suffix or extension or ''
if prefix and suffix:
file_name = prefix + suffix
input_file_name = os.path.join(workdir, file_name)
# build tempfile in case of duplicates
if os.path.exists(input_file_name):
input_file_name = tempfile.mkstemp(suffix=suffix,
prefix=prefix + '_',
dir=workdir)[1]
return input_file_name
def process(self, inpt):
"""Subclass with the appropriate handler given the data input."""
href = inpt.get('href', None)
self.inpt = inpt
if href:
if urlparse(href).scheme == 'file':
self.file = self.file_handler(inpt)
else:
# No file download occurs here. The file content will
# only be retrieved when the file property is accessed.
self.url = self.url_handler(inpt)
else:
self.data = inpt.get('data')
def process(self, inpt):
"""Subclass with the appropriate handler given the data input."""
href = inpt.get('href', None)
self.inpt = inpt
if href:
if urlparse(href).scheme == 'file':
self.file = self.file_handler(inpt)
else:
# No file download occurs here. The file content will
# only be retrieved when the file property is accessed.
self.url = self.url_handler(inpt)
else:
self.data = inpt.get('data')
def _build_file_name(self, href=''):
"""Return a file name for the local system."""
url_path = urlparse(href).path or ''
file_name = os.path.basename(url_path).strip() or 'input'
(prefix, suffix) = os.path.splitext(file_name)
suffix = suffix or self.extension
if prefix and suffix:
file_name = prefix + suffix
input_file_name = os.path.join(self.workdir, file_name)
# build tempfile in case of duplicates
if os.path.exists(input_file_name):
input_file_name = tempfile.mkstemp(suffix=suffix,
prefix=prefix + '_',
dir=self.workdir)[1]
return input_file_name
def _build_file_name(self, href=''):
"""Return a file name for the local system."""
url_path = urlparse(href).path or ''
file_name = os.path.basename(url_path).strip() or 'input'
(prefix, suffix) = os.path.splitext(file_name)
suffix = suffix or self.extension
if prefix and suffix:
file_name = prefix + suffix
input_file_name = os.path.join(self.workdir, file_name)
# build tempfile in case of duplicates
if os.path.exists(input_file_name):
input_file_name = tempfile.mkstemp(
suffix=suffix, prefix=prefix + '_',
dir=self.workdir)[1]
return input_file_name
def _validate_file_input(href):
href = href or ''
parsed_url = urlparse(href)
if parsed_url.scheme != 'file':
raise FileURLNotSupported('Invalid URL scheme')
file_path = parsed_url.path
if not file_path:
raise FileURLNotSupported('Invalid URL path')
file_path = os.path.abspath(file_path)
# build allowed paths list
inputpaths = config.get_config_value('server', 'allowedinputpaths')
allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(':') if p.strip()]
for allowed_path in allowed_paths:
if file_path.startswith(allowed_path):
LOGGER.debug("Accepted file url as input.")
return
raise FileURLNotSupported()
def _validate_file_input(href):
href = href or ''
parsed_url = urlparse(href)
if parsed_url.scheme != 'file':
raise FileURLNotSupported('Invalid URL scheme')
file_path = parsed_url.path
if not file_path:
raise FileURLNotSupported('Invalid URL path')
file_path = os.path.abspath(file_path)
# build allowed paths list
inputpaths = config.get_config_value('server', 'allowedinputpaths')
allowed_paths = [os.path.abspath(p.strip()) for p in inputpaths.split(':') if p.strip()]
for allowed_path in allowed_paths:
if file_path.startswith(allowed_path):
LOGGER.debug("Accepted file url as input.")
return
raise FileURLNotSupported()
def file_handler(complexinput, datain):
"""<wps:Reference /> handler.
Used when href is a file url."""
# save the file reference input in workdir
tmp_file = _build_input_file_name(
href=datain.get('href'),
workdir=complexinput.workdir,
extension=_extension(complexinput))
try:
inpt_file = urlparse(datain.get('href')).path
os.symlink(inpt_file, tmp_file)
LOGGER.debug("Linked input file %s to %s.", inpt_file,
tmp_file)
except Exception as e:
raise NoApplicableCode("Could not link file reference: %s" % e)
complexinput.file = tmp_file
complexinput.url = datain.get('href')
complexinput.as_reference = True
def file_handler(self, inpt):
"""<wps:Reference /> handler.
Used when href is a file url."""
extend_instance(self, FileHandler)
# check if file url is allowed
self._validate_file_input(href=inpt.get('href'))
# save the file reference input in workdir
tmp_file = self._build_file_name(href=inpt.get('href'))
try:
inpt_file = urlparse(inpt.get('href')).path
inpt_file = os.path.abspath(inpt_file)
os.symlink(inpt_file, tmp_file)
LOGGER.debug("Linked input file %s to %s.", inpt_file, tmp_file)
except Exception:
# TODO: handle os.symlink on windows
# raise NoApplicableCode("Could not link file reference: %s" % e)
LOGGER.warn("Could not link file reference")
shutil.copy2(inpt_file, tmp_file)
return tmp_file
def file_handler(self, inpt):
"""<wps:Reference /> handler.
Used when href is a file url."""
extend_instance(self, FileHandler)
# check if file url is allowed
self._validate_file_input(href=inpt.get('href'))
# save the file reference input in workdir
tmp_file = self._build_file_name(href=inpt.get('href'))
try:
inpt_file = urlparse(inpt.get('href')).path
inpt_file = os.path.abspath(inpt_file)
os.symlink(inpt_file, tmp_file)
LOGGER.debug("Linked input file {} to {}.".format(inpt_file, tmp_file))
except Exception:
# TODO: handle os.symlink on windows
# raise NoApplicableCode("Could not link file reference: {}".format(e))
LOGGER.warn("Could not link file reference")
shutil.copy2(inpt_file, tmp_file)
return tmp_file
def _get_complex_input_handler(self, href):
"""Return function for parsing and storing complexdata
:param href: href object yes or not
"""
def href_handler(complexinput, datain):
"""<wps:Reference /> handler"""
# save the reference input in workdir
tmp_file = _build_input_file_name(
href=datain.get('href'),
workdir=complexinput.workdir,
extension=_extension(complexinput))
try:
reference_file = _openurl(datain)
data_size = reference_file.headers.get('Content-Length', 0)
except Exception as e:
raise NoApplicableCode('File reference error: %s' % e)
# if the response did not return a 'Content-Length' header then
# calculate the size
if data_size == 0:
LOGGER.debug('no Content-Length, calculating size')
# check if input file size was not exceeded
complexinput.calculate_max_input_size()
max_byte_size = complexinput.max_size * 1024 * 1024
if int(data_size) > int(max_byte_size):
raise FileSizeExceeded(
'File size for input exceeded.'
' Maximum allowed: %i megabytes' % complexinput.max_size,
complexinput.identifier)
try:
with open(tmp_file, 'wb') as f:
data_size = 0
for chunk in reference_file.iter_content(chunk_size=1024):
data_size += len(chunk)
if int(data_size) > int(max_byte_size):
raise FileSizeExceeded(
'File size for input exceeded.'
' Maximum allowed: %i megabytes' %
complexinput.max_size, complexinput.identifier)
f.write(chunk)
except Exception as e:
raise NoApplicableCode(e)
complexinput.file = tmp_file
complexinput.url = datain.get('href')
complexinput.as_reference = True
def file_handler(complexinput, datain):
"""<wps:Reference /> handler.
Used when href is a file url."""
# check if file url is allowed
_validate_file_input(href=datain.get('href'))
# save the file reference input in workdir
tmp_file = _build_input_file_name(
href=datain.get('href'),
workdir=complexinput.workdir,
extension=_extension(complexinput))
try:
inpt_file = urlparse(datain.get('href')).path
inpt_file = os.path.abspath(inpt_file)
os.symlink(inpt_file, tmp_file)
LOGGER.debug("Linked input file %s to %s.", inpt_file,
tmp_file)
except Exception:
# TODO: handle os.symlink on windows
# raise NoApplicableCode("Could not link file reference: %s" % e)
LOGGER.warn("Could not link file reference")
shutil.copy2(inpt_file, tmp_file)
complexinput.file = tmp_file
complexinput.url = datain.get('href')
complexinput.as_reference = True
def data_handler(complexinput, datain):
"""<wps:Data> ... </wps:Data> handler"""
complexinput.data = datain.get('data')
if href:
if urlparse(href).scheme == 'file':
return file_handler
else:
return href_handler
else:
return data_handler