def test_http_request_post():
# here we replace owslib.util.http_post with a mock object
# because we are not interested in testing owslib
method = "POST"
url = "some_phony_url"
request = "some_phony_request"
timeout = 40
with mock.patch("pycsw.core.util.http_post",
autospec=True) as mock_http_post:
util.http_request(method=method,
url=url,
request=request,
timeout=timeout)
mock_http_post.assert_called_with(url, request, timeout=timeout)
def _parse_waf(context, repos, record, identifier):
recobjs = []
content = util.http_request('GET', record)
LOGGER.debug(content)
try:
parser = etree.HTMLParser()
tree = etree.fromstring(content, parser)
except Exception as err:
raise Exception('Could not parse WAF: %s' % str(err))
up = urlparse(record)
links = []
LOGGER.debug('collecting links')
for link in tree.xpath('//a/@href'):
link = link.strip()
if not link:
continue
if link.find('?') != -1:
continue
if not link.endswith('.xml'):
LOGGER.debug('Skipping, not .xml')
continue
if '/' in link: # path is embedded in link
if link[-1] == '/': # directory, skip
continue
if link[0] == '/':
# strip path of WAF URL
link = '%s://%s%s' % (up.scheme, up.netloc, link)
else: # tack on href to WAF URL
link = '%s/%s' % (record, link)
LOGGER.debug('URL is: %s', link)
links.append(link)
LOGGER.debug('%d links found', len(links))
for link in links:
LOGGER.debug('Processing link %s', link)
# fetch and parse
linkcontent = util.http_request('GET', link)
recobj = _parse_metadata(context, repos, linkcontent)[0]
recobj.source = link
recobj.mdsource = link
recobjs.append(recobj)
return recobjs
def test_http_request_post():
# here we replace owslib.util.http_post with a mock object
# because we are not interested in testing owslib
method = "POST"
url = "some_phony_url"
request = "some_phony_request"
timeout = 40
with mock.patch("pycsw.core.util.http_post",
autospec=True) as mock_http_post:
util.http_request(
method=method,
url=url,
request=request,
timeout=timeout
)
mock_http_post.assert_called_with(url, request, timeout=timeout)
if sfile == 'requests.txt': # GET requests
filename = '%s%s%s' % (root, os.sep, sfile)
with open(filename) as f:
gets = csv.reader(f)
for row in gets:
testfile = '%s%s%s' % (root, os.sep, sfile)
request = ','.join(row[1:]).replace(
'PYCSW_SERVER', URL)
outfile = '%s%s' % (root.replace(
os.sep, '_'), '_%s.xml' % row[0])
expected = 'expected%s%s' % (os.sep, outfile)
print('\n test %s:%s' % (testfile, row[0]))
try:
result = http_request('GET', request)
except Exception as err:
result = err.read()
status = get_validity(expected, result,
outfile, force_id_mask)
if status == 1:
print(' passed')
PASSED += 1
elif status == 0:
print(' initialized')
INITED += 1
elif status == -1 and DATABASE == 'PostgreSQL':
print(
' warning: possible collation issue')
if sfile == 'requests.txt': # GET requests
filename = '%s%s%s' % (root, os.sep, sfile)
with open(filename) as f:
gets = csv.reader(f)
for row in gets:
testfile = '%s%s%s' % (root, os.sep, sfile)
request = ','.join(row[1:]).replace('PYCSW_SERVER',
URL)
outfile = '%s%s' % (root.replace(os.sep, '_'),
'_%s.xml' % row[0])
expected = 'expected%s%s' % (os.sep, outfile)
print('\n test %s:%s' % (testfile, row[0]))
try:
result = http_request('GET', request)
except Exception as err:
result = err.read()
status = get_validity(expected, result, outfile,
force_id_mask)
if status == 1:
print(' passed')
PASSED += 1
elif status == 0:
print(' initialized')
INITED += 1
elif status == -1 and DATABASE == 'PostgreSQL':
print(' warning: possible collation issue')
WARNING += 1
def parse_record(context, record, repos=None,
mtype='http://www.opengis.net/cat/csw/2.0.2',
identifier=None, pagesize=10):
''' parse metadata '''
if identifier is None:
identifier = uuid.uuid4().get_urn()
# parse web services
if (mtype == 'http://www.opengis.net/cat/csw/2.0.2' and
isinstance(record, str) and record.startswith('http')):
LOGGER.debug('CSW service detected, fetching via HTTP')
# CSW service, not csw:Record
try:
return _parse_csw(context, repos, record, identifier, pagesize)
except Exception as err:
# TODO: implement better exception handling
if str(err).find('ExceptionReport') != -1:
msg = 'CSW harvesting error: %s' % str(err)
LOGGER.debug(msg)
raise RuntimeError(msg)
LOGGER.debug('Not a CSW, attempting to fetch Dublin Core')
try:
content = util.http_request('GET', record)
except Exception as err:
raise RuntimeError('HTTP error: %s' % str(err))
return [_parse_dc(context, repos, etree.fromstring(content, context.parser))]
elif mtype == 'urn:geoss:waf': # WAF
LOGGER.debug('WAF detected, fetching via HTTP')
return _parse_waf(context, repos, record, identifier)
elif mtype == 'http://www.opengis.net/wms': # WMS
LOGGER.debug('WMS detected, fetching via OWSLib')
return _parse_wms(context, repos, record, identifier)
elif mtype == 'http://www.opengis.net/wps/1.0.0': # WPS
LOGGER.debug('WPS detected, fetching via OWSLib')
return [_parse_wps(context, repos, record, identifier)]
elif mtype == 'http://www.opengis.net/wfs': # WFS
LOGGER.debug('WFS detected, fetching via OWSLib')
return _parse_wfs(context, repos, record, identifier)
elif mtype == 'http://www.opengis.net/wcs': # WCS
LOGGER.debug('WCS detected, fetching via OWSLib')
return _parse_wcs(context, repos, record, identifier)
elif mtype == 'http://www.opengis.net/sos/1.0': # SOS 1.0.0
LOGGER.debug('SOS 1.0.0 detected, fetching via OWSLib')
return _parse_sos(context, repos, record, identifier, '1.0.0')
elif mtype == 'http://www.opengis.net/sos/2.0': # SOS 2.0.0
LOGGER.debug('SOS 2.0.0 detected, fetching via OWSLib')
return _parse_sos(context, repos, record, identifier, '2.0.0')
elif (mtype == 'http://www.opengis.net/cat/csw/csdgm' and
record.startswith('http')): # FGDC
LOGGER.debug('FGDC detected, fetching via HTTP')
record = util.http_request('GET', record)
return _parse_metadata(context, repos, record)
