def wait_for_slot(send_queue):
print('get status')
try:
status = overpass.get_status()
except overpass.OverpassError as e:
r = e.args[0]
body = f'URL: {r.url}\n\nresponse:\n{r.text}'
mail.send_mail('Overpass API unavailable', body)
send_queue.put({'type': 'error', 'msg': "Can't access overpass API"})
return False
except requests.exceptions.Timeout:
body = 'Timeout talking to overpass API'
mail.send_mail('Overpass API timeout', body)
send_queue.put({'type': 'error', 'msg': "Can't access overpass API"})
return False
print('status:', status)
if not status['slots']:
return True
secs = status['slots'][0]
if secs <= 0:
return True
send_queue.put({'type': 'status', 'wait': secs})
sleep(secs)
return True
def wait_for_slot(send_queue):
print("get status")
try:
status = overpass.get_status()
except overpass.OverpassError as e:
r = e.args[0]
body = f"URL: {r.url}\n\nresponse:\n{r.text}"
mail.send_mail("Overpass API unavailable", body)
send_queue.put({"type": "error", "msg": "Can't access overpass API"})
return False
except requests.exceptions.Timeout:
body = "Timeout talking to overpass API"
mail.send_mail("Overpass API timeout", body)
send_queue.put({"type": "error", "msg": "Can't access overpass API"})
return False
print("status:", status)
if not status["slots"]:
return True
secs = status["slots"][0]
if secs <= 0:
return True
send_queue.put({"type": "status", "wait": secs})
sleep(secs)
return True
def matcher(self):
place = self.place
self.get_items()
db_items = {item.qid: item for item in self.place.items}
item_count = len(db_items)
self.status('{:,d} Wikidata items found'.format(item_count))
self.check_for_stop()
self.get_item_detail(db_items)
chunk_size = 96 if self.want_isa else None
skip = {'building', 'building=yes'} if self.want_isa else set()
if place.osm_type == 'node':
oql = place.get_oql()
chunks = [{
'filename': f'{place.place_id}.xml',
'num': 0,
'oql': oql
}]
else:
chunks = place.get_chunks(chunk_size=chunk_size, skip=skip)
self.report_empty_chunks(chunks)
self.check_for_stop()
overpass_good = self.overpass_request(chunks)
assert overpass_good
self.check_for_stop()
overpass_dir = app.config['OVERPASS_DIR']
for chunk in chunks:
self.check_for_stop()
if not chunk['oql']:
continue # empty chunk
filename = os.path.join(overpass_dir, chunk['filename'])
if (os.path.getsize(filename) > 2000
or "<remark> runtime error" not in open(filename).read()):
continue
root = lxml.etree.parse(filename).getroot()
remark = root.find('.//remark')
self.error('overpass: '******'Overpass error', remark.text)
return # FIXME report error to admin
if len(chunks) > 1:
self.merge_chunks(chunks)
self.check_for_stop()
self.run_osm2pgsql()
self.check_for_stop()
self.load_isa()
self.check_for_stop()
self.run_matcher()
self.check_for_stop()
self.place.clean_up()
def overpass_chunk_error(self, chunk):
if not chunk["oql"]:
return # empty chunk
filename = overpass_chunk_filename(chunk)
if not error_in_overpass_chunk(filename):
return
self.check_for_stop()
root = lxml.etree.parse(filename).getroot()
remark = root.find(".//remark")
self.error("overpass: "******"Overpass error", remark.text)
return True # FIXME report error to admin