def schema(self):
'''Return a schema to validate against, anzlic/gmd'''
schema = None
try:
with open(ANZLIC) as scontent:
sdoc = LXMLetree().parse(scontent).gettree()
schema = LXMLetree().XMLSchema(sdoc)
except XMLSchemaParseError as xse:
print(
'WFSMeta XMLSchemaParseError causing Failure - {0}\nThis usually indicates a libxml2<2.7.8 problem'
.format(xse))
self.checklog.error('WFSxs - {0}'.format(xse))
return schema
def parse(self, pno):
'''Parses useful date fields from LDS layer metadata XML'''
retry = True
ecounter = 0
n1, n2, n3, n4 = 4 * (0, )
er = None
url = 'https://data.linz.govt.nz/layer/{0}/metadata/iso/xml'.format(
pno)
dd = {}
while retry:
retry = False
content = None
try:
#tree = LXMLetree().parse(url)
with closing(U2.urlopen(url)) as content:
tree = LXMLetree().parse(content)
n1 = tree.getroot().find(mxp['fd'], namespaces=NSX).text
n2 = tree.getroot().find(mxp['pd'], namespaces=NSX).text
if n2 and not (re.search(
tree.getroot().find(mxp['pdt'], namespaces=NSX),
'publication')):
raise DateAvailabilityException(
'Cannot associate date to datetype Pub')
n3 = tree.getroot().find(mxp['rd'], namespaces=NSX).text
if n3 and not (re.search(
tree.getroot().find(mxp['rdt'], namespaces=NSX),
'revision')):
raise DateAvailabilityException(
'Cannot associate date to datetype Rev')
n4 = tree.getroot().find(mxp['ed'], namespaces=NSX).text
except U2.HTTPError as he:
if ecounter > MAX_DVL_RETRY:
raise
else:
self.checklog.warn('LPS1 - Error parsing metadata dates ' +
pno)
ecounter += 1
retry = True
except DateAvailabilityException as dae:
self.checklog.warn(
'LPS2 - Error identifying date {0}. {1}'.format(pno, dae))
finally:
if content: content.close()
return n1, n2, n3, n4, er
def metadata(self, lid):
b64s = base64.encodestring('{0}:{1}'.format(*self.userpass)).replace(
'\n', '')
svr = None
url = self.mta.format(lid=lid)
self.checklog.debug('WFS0u-' + kmask(url))
retry = True
while retry:
retry = False
content = None
try:
#cookiesetup(url)
req = U2.Request(url)
req.add_header('Authorization', 'Basic {0}'.format(b64s))
with closing(U2.urlopen(url=req,
timeout=URL_TIMEOUT)) as content:
ct = str(content.read())
if self.content(
lid, ct
): #the checkcontent here pagesscrapes for valid XML without known errors so the parser wont break
continue
xdoc = LXMLetree().fromstring(ct)
#print xdoc.getroot().find('gmd:MD_Metadata',namespaces=NSX)
if self.schema.validate(
xdoc.gettree()) and self.metamandatory(xdoc):
#if self.validate(content,schema):
self.checklog.debug('WFS0v-{0}'.format(lid))
else:
raise WFSServiceException(
'XML Fails validation against gmd.xsd with {0}'.
format(self.schema.error_log.last_error.message))
except U2.HTTPError as he:
if re.search('429', str(he)):
self.checklog.debug(
'WFS0k - Swap keys and Retry on 429. {1}'.format(
SLEEP_TIME, he))
global KEY
KEY = apikey(KEY_FILE)
if KEYINDEX == 0:
self.checklog.debug(
'WFS04 - Wait {0}s and Retry on 429. {1}'.format(
SLEEP_TIME, he))
time.sleep(SLEEP_TIME)
retry = True
continue
print('WFSMeta {0} Failure - {1}'.format(lid, he))
self.checklog.error(
'WFSm0h - {0}/{1}\n{2} on server {3}'.format(
lid, he, kmask(url), svr))
except WFSServiceException as we:
print('WFSMeta {0} XSD Failure - {1}'.format(lid, we))
self.checklog.error(
'WFSm0se - {0}/{1}\n{2} on server {3}'.format(
lid, we, kmask(url), svr))
except XMLSyntaxError as xe:
print('WFSMeta {0} XMLSyntaxError causing Parse Failure - {1}'.
format(lid, xe))
self.checklog.error(
'WFSm0x - {0}/{1}\n{2}\n{3} on server {4}'.format(
lid, xe, kmask(url), ct[:100], svr))
except U2.URLError as ue:
print('WFS0ue {0} URLError - {1}'.format(lid, ue))
if isinstance(ue.reason, socket.timeout):
self.checklog.error(
'WFS0uet - {0}/{1}\n{2} on server {3}. Retry'.format(
lid, ue, kmask(url), svr))
retry = True
continue
else:
self.checklog.error(
'WFS0ueo - {0}/{1}\n{2} on server {3}'.format(
lid, ue, kmask(url), svr))
except Exception as ee:
print('WFS0ee {0} Exception - {1}'.format(lid, ee))
self.checklog.error(
'WFS0ee - {0}/{1}\n{2} on server {3}'.format(
lid, ee, kmask(url), svr))
def __init__(self, upd, checklog):
self.userpass = upd
self.checklog = checklog
self.tree = LXMLetree().parse(
self.rs3.format(id=id[0].text, csw_v=CSW, gmd=NSX['gmd']))
self.schema = self.schema()
