def fetch_zip(url, output_path, feature_type, progress_dialog=None):
"""Download zip containing shp file and write to output_path.
.. versionadded:: 3.2
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param progress_dialog: A progress dialog.
:type progress_dialog: QProgressDialog
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
if progress_dialog:
progress_dialog.show()
# Infinite progress bar when the server is fetching data.
# The progress bar will be updated with the file size later.
progress_dialog.setMaximum(0)
progress_dialog.setMinimum(0)
progress_dialog.setValue(0)
# Get a pretty label from feature_type, but not translatable
label_feature_type = feature_type.replace('-', ' ')
label_text = tr('Fetching %s' % label_feature_type)
progress_dialog.setLabelText(label_text)
# Set Proxy in web page
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if proxy is not None:
network_manager.setProxy(proxy)
# Download Process
downloader = FileDownloader(
network_manager, url, output_path, progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
if result[0] == QNetworkReply.OperationCanceledError:
raise CanceledImportDialogError(error_message)
else:
raise DownloadError(error_message)
def fetch_zip(url, output_path, feature_type, progress_dialog=None):
"""Download zip containing shp file and write to output_path.
.. versionadded:: 3.2
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param progress_dialog: A progress dialog.
:type progress_dialog: QProgressDialog
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
if progress_dialog:
progress_dialog.show()
# Infinite progress bar when the server is fetching data.
# The progress bar will be updated with the file size later.
progress_dialog.setMaximum(0)
progress_dialog.setMinimum(0)
progress_dialog.setValue(0)
# Get a pretty label from feature_type, but not translatable
label_feature_type = feature_type.replace('-', ' ')
label_text = tr('Fetching %s' % label_feature_type)
progress_dialog.setLabelText(label_text)
# Set Proxy in web page
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if proxy is not None:
network_manager.setProxy(proxy)
# Download Process
downloader = FileDownloader(network_manager, url, output_path,
progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
if result[0] == QNetworkReply.OperationCanceledError:
raise CanceledImportDialogError(error_message)
else:
raise DownloadError(error_message)
def setProxy(self, proxy):
"""Allow setting string as proxy
"""
fragments = common.parse_proxy(proxy)
if fragments.host:
QNetworkAccessManager.setProxy(self,
QNetworkProxy(QNetworkProxy.HttpProxy,
fragments.host, int(fragments.port),
fragments.username, fragments.password
)
)
else:
common.logger.info('Invalid proxy:' + proxy)
proxy = None
def setProxy(self, proxy):
"""Parse proxy components from proxy
"""
if proxy:
fragments = common.parse_proxy(proxy)
if fragments['host']:
QNetworkAccessManager.setProxy(
self,
QNetworkProxy(QNetworkProxy.HttpProxy, fragments['host'],
int(fragments['port']),
fragments['username'],
fragments['password']))
else:
common.logger.info('Invalid proxy: ' + str(proxy))
def download_from_url(url, output_directory, filename=None, use_cache=True):
"""Download file from a url and put it under output_directory.
:param url: Url that gives response.
:type url: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param filename: Optional filename for downloaded file.
:type filename: str
:param use_cache: If there is a cached copy of the file already in the
output directory, do not refetch it (True) or force refecth it (False).
:type use_cache: bool
:returns: File path if success to download, else None
:rtype: str
"""
if filename is None:
filename = get_filename(url)
LOGGER.info('Download file %s from %s' % (filename, url))
file_path = os.path.join(output_directory, filename)
if os.path.exists(file_path) and use_cache:
LOGGER.info('File %s exists, not downloading' % file_path)
return file_path
# Set Proxy in webpage
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if not proxy is None:
network_manager.setProxy(proxy)
# Download Process
# noinspection PyTypeChecker
downloader = FileDownloader(network_manager, url, file_path)
try:
result = downloader.download()
except IOError as ex:
raise DownloadException(ex)
if result[0] is not True:
_, error_message = result
raise DownloadException(error_message)
if os.path.exists(file_path):
return file_path
else:
return None
def download_from_url(url, output_directory, filename=None, use_cache=True):
"""Download file from a url and put it under output_directory.
:param url: Url that gives response.
:type url: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param filename: Optional filename for downloaded file.
:type filename: str
:param use_cache: If there is a cached copy of the file already in the
output directory, do not refetch it (True) or force refecth it (False).
:type use_cache: bool
:returns: File path if success to download, else None
:rtype: str
"""
if filename is None:
filename = get_filename(url)
LOGGER.info('Download file %s from %s' % (filename, url))
file_path = os.path.join(output_directory, filename)
if os.path.exists(file_path) and use_cache:
LOGGER.info('File %s exists, not downloading' % file_path)
return file_path
# Set Proxy in webpage
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if not proxy is None:
network_manager.setProxy(proxy)
# Download Process
# noinspection PyTypeChecker
downloader = FileDownloader(network_manager, url, file_path)
try:
result = downloader.download()
except IOError as ex:
raise DownloadException(ex)
if result[0] is not True:
_, error_message = result
raise DownloadException(error_message)
if os.path.exists(file_path):
return file_path
else:
return None
def setProxy(self, proxy):
"""Allow setting string as proxy
"""
if proxy:
fragments = common.parse_proxy(proxy)
if fragments.host:
QNetworkAccessManager.setProxy(self,
QNetworkProxy(QNetworkProxy.HttpProxy,
fragments.host, int(fragments.port),
fragments.username, fragments.password
)
)
else:
common.logger.info('Invalid proxy: ' + str(proxy))
proxy = None
def setProxy(self, proxy):
"""Allow setting string as proxy
"""
if isinstance(proxy, basestring):
match = re.match('((?P<username>\w+):(?P<password>\w+)@)?(?P<host>\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})(:(?P<port>\d+))?', proxy)
if match:
groups = match.groupdict()
username = groups.get('username') or ''
password = groups.get('password') or ''
host = groups.get('host')
port = groups.get('port')
#print host, port, username, password
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, host, int(port), username, password)
else:
common.logger.info('Invalid proxy:' + proxy)
proxy = None
if proxy:
QNetworkAccessManager.setProxy(self, proxy)
def test_access_without_credentials(self):
loop = QEventLoop()
proxy = get_network_proxy()
manager = QNetworkAccessManager()
manager.setProxy(proxy)
manager.finished.connect(loop.exit)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
loop.exec_(flags=QEventLoop.ExcludeUserInputEvents)
if reply.isFinished():
self.assertEquals(self.server.log.getvalue(),
'407 Proxy Authentication Required\n\n')
else:
if reply.isRunning():
self.failUnless(False, msg='The request has timed out.')
else:
self.failUnless(False, msg='A Network error occurred.')
def test_access_without_credentials(self):
loop = QEventLoop()
proxy = get_network_proxy()
manager = QNetworkAccessManager()
manager.setProxy(proxy)
manager.finished.connect(loop.exit)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
loop.exec_(flags=QEventLoop.ExcludeUserInputEvents)
if reply.isFinished():
self.assertEquals(self.server.log.getvalue(),
'407 Proxy Authentication Required\n\n')
else:
if reply.isRunning():
self.failUnless(False, msg='The request has timed out.')
else:
self.failUnless(False, msg='A Network error occurred.')
def test_access_to_remote_succeeded(self):
loop = QEventLoop()
proxy = get_network_proxy()
proxy.setUser(self.server.username)
proxy.setPassword(self.server.password)
manager = QNetworkAccessManager()
manager.setProxy(proxy)
manager.finished.connect(loop.exit)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
loop.exec_(flags=QEventLoop.ExcludeUserInputEvents)
if reply.isFinished():
response_code = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute).toString()
self.assertEquals(response_code, '200')
self.assertEquals(reply.url(), QUrl('http://aws.amazon.com/'))
else:
if reply.isRunning():
self.failUnless(False, msg='The request has timed out.')
else:
self.failUnless(False, msg='A Network error occurred.')
def test_access_to_remote_succeeded(self):
loop = QEventLoop()
proxy = get_network_proxy()
proxy.setUser(self.server.username)
proxy.setPassword(self.server.password)
manager = QNetworkAccessManager()
manager.setProxy(proxy)
manager.finished.connect(loop.exit)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
loop.exec_(flags=QEventLoop.ExcludeUserInputEvents)
if reply.isFinished():
response_code = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute).toString()
self.assertEquals(response_code, '200')
self.assertEquals(reply.url(), QUrl('http://aws.amazon.com/'))
else:
if reply.isRunning():
self.failUnless(False, msg='The request has timed out.')
else:
self.failUnless(False, msg='A Network error occurred.')
class PageBrower(QWebPage):
def __init__(self):
QWebPage.__init__(self)
self.networkAccessManager = QNetworkAccessManager()
self.networkNoProxy = QNetworkProxy(QNetworkProxy.NoProxy)
self.networkProxy = QNetworkProxy(QNetworkProxy.HttpProxy, "127.0.0.1",
8087)
self.setNetworkAccessManager(self.networkAccessManager)
self.loadFinished.connect(self._loadFinished)
def installProxy(self):
self.networkAccessManager.setProxy(self.networkProxy)
def uninstallProxt(self):
self.networkAccessManager.setProxy(self.networkNoProxy)
def getErrorCode(self, url):
ret = False
req = urllib2.Request(url)
try:
urllib2.urlopen(url)
except IOError, e:
if hasattr(e, 'reason'):
print "%s failed to be reached due to %s" % (url, e.reason)
ret = True
elif hasattr(e, 'code'):
print " %d :The server could not fulfill the request" % (
e.code)
ret = True
else:
print "unknow error type"
ret = True
return ret
class PageBrower(QWebPage):
def __init__(self):
QWebPage.__init__(self)
self.networkAccessManager = QNetworkAccessManager()
self.networkNoProxy = QNetworkProxy(QNetworkProxy.NoProxy)
self.networkProxy = QNetworkProxy(QNetworkProxy.HttpProxy,"127.0.0.1",8087)
self.setNetworkAccessManager(self.networkAccessManager)
self.loadFinished.connect(self._loadFinished)
def installProxy(self):
self.networkAccessManager.setProxy(self.networkProxy)
def uninstallProxt(self):
self.networkAccessManager.setProxy(self.networkNoProxy)
def getErrorCode(self,url):
ret = False
req = urllib2.Request(url)
try:
urllib2.urlopen(url)
except IOError, e:
if hasattr(e,'reason'):
print "%s failed to be reached due to %s" % (url,e.reason)
ret = True
elif hasattr(e,'code'):
print " %d :The server could not fulfill the request" % (e.code)
ret = True
else:
print "unknow error type"
ret = True
return ret
class WebClient(BaseWebClient):
"""A webclient with a qtnetwork backend."""
proxy_instance = None
def __init__(self, *args, **kwargs):
"""Initialize this instance."""
super(WebClient, self).__init__(*args, **kwargs)
self.nam = QNetworkAccessManager(QCoreApplication.instance())
self.nam.finished.connect(self._handle_finished)
self.nam.authenticationRequired.connect(self._handle_authentication)
self.nam.proxyAuthenticationRequired.connect(self.handle_proxy_auth)
self.nam.sslErrors.connect(self._handle_ssl_errors)
self.replies = {}
self.proxy_retry = False
self.setup_proxy()
# Force Qt to load the system certificates
QSslSocket.setDefaultCaCertificates(QSslSocket.systemCaCertificates())
# Apply our local certificates as the SSL configuration to be used
# for all QNetworkRequest calls.
self.ssl_config = QSslConfiguration.defaultConfiguration()
ca_certs = self.ssl_config.caCertificates()
try:
for path in glob.glob(os.path.join(get_cert_dir(),
"UbuntuOne*.pem")):
with open(path) as f:
cert = QSslCertificate(f.read())
if cert.isValid():
ca_certs.append(cert)
else:
logger.error("invalid certificate: {}".format(path))
except (IndexError, IOError) as err:
raise WebClientError(
"Unable to configure SSL certificates: {}".format(err))
self.ssl_config.setCaCertificates(ca_certs)
def _set_proxy(self, proxy):
"""Set the proxy to be used."""
QNetworkProxy.setApplicationProxy(proxy)
self.nam.setProxy(proxy)
def setup_proxy(self):
"""Setup the proxy settings if needed."""
# QtNetwork knows how to use the system settings on both Win and Mac
if sys.platform.startswith("linux"):
settings = gsettings.get_proxy_settings()
enabled = len(settings) > 0
if enabled and WebClient.proxy_instance is None:
proxy = build_proxy(settings)
self._set_proxy(proxy)
WebClient.proxy_instance = proxy
elif enabled and WebClient.proxy_instance:
logger.info("Proxy already in use.")
else:
logger.info("Proxy is disabled.")
else:
if WebClient.proxy_instance is None:
logger.info("Querying OS for proxy.")
QNetworkProxyFactory.setUseSystemConfiguration(True)
def handle_proxy_auth(self, proxy, authenticator):
"""Proxy authentication is required."""
logger.info("auth_required %r, %r", self.proxy_username,
proxy.hostName())
if (self.proxy_username is not None and
self.proxy_username != str(authenticator.user())):
authenticator.setUser(self.proxy_username)
WebClient.proxy_instance.setUser(self.proxy_username)
if (self.proxy_password is not None and
self.proxy_password != str(authenticator.password())):
authenticator.setPassword(self.proxy_password)
WebClient.proxy_instance.setPassword(self.proxy_password)
def _perform_request(self, request, method, post_buffer):
"""Return a deferred that will be fired with a Response object."""
d = defer.Deferred()
if method == "GET":
reply = self.nam.get(request)
elif method == "HEAD":
reply = self.nam.head(request)
else:
reply = self.nam.sendCustomRequest(request, method, post_buffer)
self.replies[reply] = d
return d
@defer.inlineCallbacks
def request(self, iri, method="GET", extra_headers=None,
oauth_credentials=None, post_content=None):
"""Return a deferred that will be fired with a Response object."""
uri = self.iri_to_uri(iri)
request = QNetworkRequest(QUrl(uri))
request.setSslConfiguration(self.ssl_config)
headers = yield self.build_request_headers(uri, method, extra_headers,
oauth_credentials)
for key, value in headers.items():
request.setRawHeader(key, value)
post_buffer = QBuffer()
post_buffer.setData(post_content)
try:
result = yield self._perform_request(request, method, post_buffer)
except ProxyUnauthorizedError as e:
app_proxy = QNetworkProxy.applicationProxy()
proxy_host = app_proxy.hostName() if app_proxy else "proxy server"
got_creds = yield self.request_proxy_auth_credentials(
proxy_host, self.proxy_retry)
if got_creds:
self.proxy_retry = True
result = yield self.request(iri, method, extra_headers,
oauth_credentials, post_content)
else:
excp = WebClientError('Proxy creds needed.', e)
defer.returnValue(excp)
defer.returnValue(result)
def _handle_authentication(self, reply, authenticator):
"""The reply needs authentication."""
if authenticator.user() != self.username:
authenticator.setUser(self.username)
if authenticator.password() != self.password:
authenticator.setPassword(self.password)
def _handle_finished(self, reply):
"""The reply has finished processing."""
assert reply in self.replies
d = self.replies.pop(reply)
error = reply.error()
content = reply.readAll()
if not error:
headers = HeaderDict()
for key, value in reply.rawHeaderPairs():
headers[str(key)].append(str(value))
response = Response(bytes(content), headers)
d.callback(response)
else:
content = unicode(content)
error_string = unicode(reply.errorString())
logger.debug('_handle_finished error (%s,%s).', error,
error_string)
if error == QNetworkReply.AuthenticationRequiredError:
exception = UnauthorizedError(error_string, content)
elif error == QNetworkReply.ProxyAuthenticationRequiredError:
# we are going thru a proxy and we did not auth
exception = ProxyUnauthorizedError(error_string, content)
else:
exception = WebClientError(error_string, content)
d.errback(exception)
def _get_certificate_details(self, cert):
"""Return an string with the details of the certificate."""
detail_titles = {QSslCertificate.Organization: 'organization',
QSslCertificate.CommonName: 'common_name',
QSslCertificate.LocalityName: 'locality_name',
QSslCertificate.OrganizationalUnitName: 'unit',
QSslCertificate.CountryName: 'country_name',
QSslCertificate.StateOrProvinceName: 'state_name'}
details = {}
for info, title in detail_titles.items():
details[title] = str(cert.issuerInfo(info))
return self.format_ssl_details(details)
def _get_certificate_host(self, cert):
"""Return the host of the cert."""
return str(cert.issuerInfo(QSslCertificate.CommonName))
def _handle_ssl_errors(self, reply, errors):
"""Handle the case in which we got an ssl error."""
msg = StringIO()
msg.write('SSL errors found; url: %s\n' %
reply.request().url().toString())
for error in errors:
msg.write('========Error=============\n%s (%s)\n' %
(error.errorString(), error.error()))
msg.write('--------Cert Details------\n%s\n' %
self._get_certificate_details(error.certificate()))
msg.write('==========================\n')
logger.error(msg.getvalue())
def force_use_proxy(self, https_settings):
"""Setup this webclient to use the given proxy settings."""
settings = {"https": https_settings}
proxy = build_proxy(settings)
self._set_proxy(proxy)
WebClient.proxy_instance = proxy
def shutdown(self):
"""Shut down all pending requests (if possible)."""
self.nam.deleteLater()
if proxy_socket.error() != QTcpSocket.RemoteHostClosedError:
url = proxy_socket.property('url').toUrl()
error_string = proxy_socket.errorString()
if self.debug:
self.log.write('Error for %s %s\n\n' % (url, error_string))
proxy_socket.deleteLater()
if __name__ == '__main__':
app = QCoreApplication(sys.argv)
server = HTTPProxy()
manager = QNetworkAccessManager()
manager.finished.connect(server.stopServing)
manager.finished.connect(app.quit)
proxy = QNetworkProxy()
proxy.setType(QNetworkProxy.HttpProxy)
proxy.setHostName('127.0.0.1')
proxy.setPort(server.port())
proxy.setUser(server.username)
proxy.setPassword(server.password)
manager.setProxy(proxy)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
sys.exit(app.exec_())
class OsmDownloader(QDialog, Ui_OsmDownloaderBase):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = "http://osm.linfiniti.com/buildings-shp"
self.roads_url = "http://osm.linfiniti.com/roads-shp"
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr("InaSAFE OpenStreetMap Downloader")
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if not proxy is None:
self.network_manager.setProxy(proxy)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent will be used to determine the area for which '
'you want data to be retrieved. You can adjust it manually using '
'the bounding box options below.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self):
""" Update extent value in GUI based from value in map."""
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads']
elif index == 1:
feature_types = ['buildings']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
self.download(feature_type)
self.load_shapefile(feature_type)
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self,
self.tr("InaSAFE OpenStreetMap downloader error"),
str(myEx))
self.progress_dialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = str(self.output_directory.text())
if os.path.exists(path):
return
title = self.tr("Directory %s not exist") % path
question = self.tr(
"Directory %s not exist. Do you want to create it?"
) % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title,
question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker, PyArgumentList
QMessageBox.warning(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type):
"""Download shapefiles from Linfinti server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError, CanceledImportDialogError
"""
## preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = (
'{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(
min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude
)
output_prefix = self.filename_prefix.text()
if feature_type == 'buildings':
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.buildings_url, box=box)
else:
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.roads_url, box=box)
if output_prefix is not None:
url += '&output_prefix=%s' % output_prefix
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path)
#print path
#print str(self.output_directory.text())
self.extract_zip(path, str(self.output_directory.text()))
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
self.progress_dialog.setMaximum(100)
self.progress_dialog.setValue(0)
# label_text = "Begin downloading shapefile from " \
# + "%s ..."
# self.progress_dialog.setLabelText(self.tr(label_text) % (url))
label_text = self.tr("Downloading shapefile")
self.progress_dialog.setLabelText(label_text)
result = download_url(
self.network_manager, url, output_path,
self.progress_dialog)
if result[0] is not True:
_, error_message = result
raise ImportDialogError(error_message)
@staticmethod
def extract_zip(path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
handle = open(path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
output_path = os.path.join(output_dir, name)
output_file = open(output_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type):
"""
Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when buildings.shp not exist
"""
output_prefix = self.filename_prefix.text()
path = str(self.output_directory.text())
path = os.path.join(path, '%s%s.shp' % (output_prefix, feature_type))
if not os.path.exists(path):
message = self.tr(
"%s don't exist. The server doesn't have any data."
)
raise ImportDialogError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
class OsmDownloader(QDialog, Ui_OsmDownloaderBase):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.url = "http://osm.linfiniti.com/buildings-shp"
# creating progress dialog for download
self.progressDialog = QProgressDialog(self)
self.progressDialog.setAutoClose(False)
myTitle = self.tr("InaSAFE OpenStreetMap Downloader")
self.progressDialog.setWindowTitle(myTitle)
# Set up context help
helpButton = self.buttonBox.button(QtGui.QDialogButtonBox.Help)
QtCore.QObject.connect(helpButton, QtCore.SIGNAL('clicked()'),
self.show_help)
self.show_info()
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if not proxy is None:
self.network_manager.setProxy(proxy)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') data from the '
'OpenStreetMap project for you. The downloaded data will have '
'InaSAFE keywords defined and a default QGIS style applied. To '
'use this tool effectively:')
tips = m.BulletedList()
tips.add(
self.
tr('Use QGIS to zoom in to the area for which you want building data '
'to be retrieved.'))
tips.add(
self.
tr('Check the output directory is correct. Note that the saved '
'dataset will be called buildings.shp (and its associated files).'
))
tips.add(
self.
tr('If a dataset already exists in the output directory it will be '
'overwritten.'))
tips.add(
self.
tr('This tool requires a working internet connection and fetching '
'buildings will consume your bandwidth.'))
tips.add(
m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.webView.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
mySetting = QSettings()
self.outDir.setText(mySetting.value('directory'))
def save_state(self):
""" Store current state of GUI to configuration file """
mySetting = QSettings()
mySetting.setValue('directory', self.outDir.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help('openstreetmap_downloader')
def update_extent(self):
""" Update extent value in GUI based from value in map."""
# Get the extent as [xmin, ymin, xmax, ymax]
myExtent = viewport_geo_array(self.iface.mapCanvas())
self.minLongitude.setText(str(myExtent[0]))
self.minLatitude.setText(str(myExtent[1]))
self.maxLongitude.setText(str(myExtent[2]))
self.maxLatitude.setText(str(myExtent[3]))
@pyqtSignature('') # prevents actions being handled twice
def on_pBtnDir_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.outDir.setText(
QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
try:
self.save_state()
self.require_directory()
self.download()
self.load_shapefile()
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self, self.tr("InaSAFE OpenStreetMap downloader error"),
str(myEx))
self.progressDialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
myDir = str(self.outDir.text())
if os.path.exists(myDir):
return
myTitle = self.tr("Directory %s not exist") % myDir
myQuestion = self.tr(
"Directory %s not exist. Do you want to create it?") % myDir
# noinspection PyCallByClass,PyTypeChecker
myAnswer = QMessageBox.question(self, myTitle, myQuestion,
QMessageBox.Yes | QMessageBox.No)
if myAnswer == QMessageBox.Yes:
os.makedirs(myDir)
else:
raise CanceledImportDialogError()
def download(self):
"""Download shapefiles from Linfinti server.
:raises: ImportDialogError, CanceledImportDialogError
"""
## preparing necessary data
myMinLng = str(self.minLongitude.text())
myMinLat = str(self.minLatitude.text())
myMaxLng = str(self.maxLongitude.text())
myMaxLat = str(self.maxLatitude.text())
myCoordinate = "{myMinLng},{myMinLat},{myMaxLng},{myMaxLat}".format(
myMinLng=myMinLng,
myMinLat=myMinLat,
myMaxLng=myMaxLng,
myMaxLat=myMaxLat)
myShapeUrl = "{url}?bbox={myCoordinate}".format(
url=self.url, myCoordinate=myCoordinate)
myFilePath = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(myShapeUrl, myFilePath)
print myFilePath
print str(self.outDir.text())
self.extract_zip(myFilePath, str(self.outDir.text()))
self.progressDialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
self.progressDialog.show()
self.progressDialog.setMaximum(100)
self.progressDialog.setValue(0)
# myLabelText = "Begin downloading shapefile from " \
# + "%s ..."
# self.progressDialog.setLabelText(self.tr(myLabelText) % (url))
myLabelText = self.tr("Downloading shapefile")
self.progressDialog.setLabelText(myLabelText)
myResult = download_url(self.network_manager, url, output_path,
self.progressDialog)
if myResult is not True:
_, myErrorMessage = myResult
raise ImportDialogError(myErrorMessage)
def extract_zip(self, path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
myHandle = open(path, 'rb')
myZip = zipfile.ZipFile(myHandle)
for myName in myZip.namelist():
myOutPath = os.path.join(output_dir, myName)
myOutFile = open(myOutPath, 'wb')
myOutFile.write(myZip.read(myName))
myOutFile.close()
myHandle.close()
def load_shapefile(self):
"""
Load downloaded shape file to QGIS Main Window.
:raises: ImportDialogError - when buildings.shp not exist
"""
myDir = str(self.outDir.text())
myPath = os.path.join(myDir, 'buildings.shp')
if not os.path.exists(myPath):
myMessage = self.tr(
"%s don't exist. The server don't have buildings data.")
raise ImportDialogError(myMessage)
self.iface.addVectorLayer(myPath, 'buildings', 'ogr')
class OsmDownloader(QDialog, Ui_OsmDownloaderBase):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.url = "http://osm.linfiniti.com/buildings-shp"
# creating progress dialog for download
self.progressDialog = QProgressDialog(self)
self.progressDialog.setAutoClose(False)
myTitle = self.tr("InaSAFE OpenStreetMap Downloader")
self.progressDialog.setWindowTitle(myTitle)
# Set up context help
helpButton = self.buttonBox.button(QtGui.QDialogButtonBox.Help)
QtCore.QObject.connect(helpButton, QtCore.SIGNAL('clicked()'),
self.show_help)
self.show_info()
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if not proxy is None:
self.network_manager.setProxy(proxy)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') data from the '
'OpenStreetMap project for you. The downloaded data will have '
'InaSAFE keywords defined and a default QGIS style applied. To '
'use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Use QGIS to zoom in to the area for which you want building data '
'to be retrieved.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called buildings.shp (and its associated files).'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.webView.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
mySetting = QSettings()
self.outDir.setText(mySetting.value('directory'))
def save_state(self):
""" Store current state of GUI to configuration file """
mySetting = QSettings()
mySetting.setValue('directory', self.outDir.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help('openstreetmap_downloader')
def update_extent(self):
""" Update extent value in GUI based from value in map."""
# Get the extent as [xmin, ymin, xmax, ymax]
myExtent = viewport_geo_array(self.iface.mapCanvas())
self.minLongitude.setText(str(myExtent[0]))
self.minLatitude.setText(str(myExtent[1]))
self.maxLongitude.setText(str(myExtent[2]))
self.maxLatitude.setText(str(myExtent[3]))
@pyqtSignature('') # prevents actions being handled twice
def on_pBtnDir_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.outDir.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
try:
self.save_state()
self.require_directory()
self.download()
self.load_shapefile()
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self,
self.tr("InaSAFE OpenStreetMap downloader error"),
str(myEx))
self.progressDialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
myDir = str(self.outDir.text())
if os.path.exists(myDir):
return
myTitle = self.tr("Directory %s not exist") % myDir
myQuestion = self.tr(
"Directory %s not exist. Do you want to create it?"
) % myDir
# noinspection PyCallByClass,PyTypeChecker
myAnswer = QMessageBox.question(
self, myTitle,
myQuestion, QMessageBox.Yes | QMessageBox.No)
if myAnswer == QMessageBox.Yes:
os.makedirs(myDir)
else:
raise CanceledImportDialogError()
def download(self):
"""Download shapefiles from Linfinti server.
:raises: ImportDialogError, CanceledImportDialogError
"""
## preparing necessary data
myMinLng = str(self.minLongitude.text())
myMinLat = str(self.minLatitude.text())
myMaxLng = str(self.maxLongitude.text())
myMaxLat = str(self.maxLatitude.text())
myCoordinate = "{myMinLng},{myMinLat},{myMaxLng},{myMaxLat}".format(
myMinLng=myMinLng,
myMinLat=myMinLat,
myMaxLng=myMaxLng,
myMaxLat=myMaxLat
)
myShapeUrl = "{url}?bbox={myCoordinate}".format(
url=self.url,
myCoordinate=myCoordinate
)
myFilePath = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(myShapeUrl, myFilePath)
print myFilePath
print str(self.outDir.text())
self.extract_zip(myFilePath, str(self.outDir.text()))
self.progressDialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
self.progressDialog.show()
self.progressDialog.setMaximum(100)
self.progressDialog.setValue(0)
# myLabelText = "Begin downloading shapefile from " \
# + "%s ..."
# self.progressDialog.setLabelText(self.tr(myLabelText) % (url))
myLabelText = self.tr("Downloading shapefile")
self.progressDialog.setLabelText(myLabelText)
myResult = download_url(
self.network_manager, url, output_path,
self.progressDialog)
if myResult is not True:
_, myErrorMessage = myResult
raise ImportDialogError(myErrorMessage)
def extract_zip(self, path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
myHandle = open(path, 'rb')
myZip = zipfile.ZipFile(myHandle)
for myName in myZip.namelist():
myOutPath = os.path.join(output_dir, myName)
myOutFile = open(myOutPath, 'wb')
myOutFile.write(myZip.read(myName))
myOutFile.close()
myHandle.close()
def load_shapefile(self):
"""
Load downloaded shape file to QGIS Main Window.
:raises: ImportDialogError - when buildings.shp not exist
"""
myDir = str(self.outDir.text())
myPath = os.path.join(myDir, 'buildings.shp')
if not os.path.exists(myPath):
myMessage = self.tr(
"%s don't exist. The server don't have buildings data."
)
raise ImportDialogError(myMessage)
self.iface.addVectorLayer(myPath, 'buildings', 'ogr')
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = 'http://osm.linfiniti.com/buildings-shp'
self.building_points_url = \
'http://osm.linfiniti.com/building-points-shp'
self.roads_url = 'http://osm.linfiniti.com/roads-shp'
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('InaSAFE OpenStreetMap Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if proxy is not None:
self.network_manager.setProxy(proxy)
self.restore_state()
# Setup the rectangle map tool
self.canvas = iface.mapCanvas()
self.rectangle_map_tool = \
RectangleMapTool(self.canvas)
self.rectangle_map_tool.rectangle_created.connect(
self.update_extent_from_rectangle)
self.button_extent_rectangle.clicked.connect(
self.drag_rectangle_on_map_canvas)
# Setup pan tool
self.pan_tool = QgsMapToolPan(self.canvas)
self.canvas.setMapTool(self.pan_tool)
self.update_extent_from_map_canvas()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:')
tips = m.BulletedList()
tips.add(
self.
tr('Your current extent, when opening this window, will be used to '
'determine the area for which you want data to be retrieved.'
'You can interactively select the area by using the '
'\'select on map\' button - which will temporarily hide this '
'window and allow you to drag a rectangle on the map. After you '
'have finished dragging the rectangle, this window will '
'reappear.'))
tips.add(
self.
tr('Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'))
tips.add(
self.
tr('By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'))
tips.add(
self.
tr('If a dataset already exists in the output directory it will be '
'overwritten.'))
tips.add(
self.
tr('This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(
m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self, extent):
"""Update extent value in GUI based from an extent.
:param extent: A list in the form [xmin, ymin, xmax, ymax] where all
coordinates provided are in Geographic / EPSG:4326.
:type extent: list
"""
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
def update_extent_from_map_canvas(self):
"""Update extent value in GUI based from value in map.
.. note:: Delegates to update_extent()
"""
self.groupBox.setTitle(self.tr('Bounding box from the map canvas'))
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.update_extent(extent)
def update_extent_from_rectangle(self):
"""Update extent value in GUI based from the QgsMapTool rectangle.
.. note:: Delegates to update_extent()
"""
self.show()
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.canvas.setMapTool(self.pan_tool)
rectangle = self.rectangle_map_tool.rectangle()
if rectangle:
self.groupBox.setTitle(self.tr('Bounding box from rectangle'))
extent = rectangle_geo_array(rectangle, self.iface.mapCanvas())
self.update_extent(extent)
def validate_extent(self):
"""Validate the bounding box before user click OK to download.
:return: True if the bounding box is valid, otherwise False
:rtype: bool
"""
min_latitude = float(str(self.min_latitude.text()))
max_latitude = float(str(self.max_latitude.text()))
min_longitude = float(str(self.min_longitude.text()))
max_longitude = float(str(self.max_longitude.text()))
# min_latitude < max_latitude
if min_latitude >= max_latitude:
return False
# min_longitude < max_longitude
if min_longitude >= max_longitude:
return False
# -90 <= latitude <= 90
if min_latitude < -90 or min_latitude > 90:
return False
if max_latitude < -90 or max_latitude > 90:
return False
# -180 <= longitude <= 180
if min_longitude < -180 or min_longitude > 180:
return False
if max_longitude < -180 or max_longitude > 180:
return False
return True
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory."""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(
QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def drag_rectangle_on_map_canvas(self):
"""Hide the dialog and allow the user to draw a rectangle."""
self.hide()
self.rectangle_map_tool.reset()
self.canvas.unsetMapTool(self.pan_tool)
self.canvas.setMapTool(self.rectangle_map_tool)
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Lock the groupbox
self.groupBox.setDisabled(True)
# Validate extent
valid_flag = self.validate_extent()
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the groupbox
self.groupBox.setEnabled(True)
return
# Get all the feature types
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads', 'building-points']
elif index == 1:
feature_types = ['buildings']
elif index == 2:
feature_types = ['building-points']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_checkBox.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, feature_type, overwrite)
self.download(feature_type, output_base_file_path)
try:
self.load_shapefile(feature_type, output_base_file_path)
except FileMissingError as exception:
display_warning_message_box(self, error_dialog_title,
exception.message)
self.done(QDialog.Accepted)
self.rectangle_map_tool.reset()
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title,
exception.message)
self.progress_dialog.cancel()
finally:
# Unlock the groupbox
self.groupBox.setEnabled(True)
def get_output_base_path(self, output_directory, output_prefix,
feature_type, overwrite):
"""Get a full base name path to save the shapefile.
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
path = os.path.join(output_directory,
'%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix('%s.shp' % path,
separator)
if suffix:
path = os.path.join(
output_directory, '%s%s%s%s' %
(output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return OsmDownloaderDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(self, title, question,
QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type, output_base_path):
"""Download shapefiles from Kartoza server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param output_base_path: The base path of the shape file.
:type output_base_path: str
:raises: ImportDialogError, CanceledImportDialogError
"""
# preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = ('{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude)
if feature_type == 'buildings':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.buildings_url, box=box)
elif feature_type == 'building-points':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.building_points_url, box=box)
else:
url = '{url}?bbox={box}&qgis_version=2'.format(url=self.roads_url,
box=box)
if 'LANG' in os.environ:
env_lang = os.environ['LANG']
url += '&lang=%s' % env_lang
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path, feature_type)
self.extract_zip(path, output_base_path)
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path, feature_type):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
# Infinite progress bar when the server is fetching data.
# The progress bar will be updated with the file size later.
self.progress_dialog.setMaximum(0)
self.progress_dialog.setMinimum(0)
self.progress_dialog.setValue(0)
# Get a pretty label from feature_type, but not translatable
label_feature_type = feature_type.replace('-', ' ')
label_text = self.tr('Fetching %s' % label_feature_type)
self.progress_dialog.setLabelText(label_text)
# Download Process
downloader = FileDownloader(self.network_manager, url, output_path,
self.progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
if result[0] == QNetworkReply.OperationCanceledError:
raise CanceledImportDialogError(error_message)
else:
raise DownloadError(error_message)
@staticmethod
def extract_zip(zip_path, destination_base_path):
"""Extract different extensions to the destination base path.
Example : test.zip contains a.shp, a.dbf, a.prj
and destination_base_path = '/tmp/CT-buildings
Expected result :
- /tmp/CT-buildings.shp
- /tmp/CT-buildings.dbf
- /tmp/CT-buildings.prj
If two files in the zip with the same extension, only one will be
copied.
:param zip_path: The path of the .zip file
:type zip_path: str
:param destination_base_path: The destination base path where the shp
will be written to.
:type destination_base_path: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
handle = open(zip_path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
extension = os.path.splitext(name)[1]
output_final_path = u'%s%s' % (destination_base_path, extension)
output_file = open(output_final_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers'))
def reject(self):
"""Redefinition of the reject() method
to remove the rectangle selection tool.
It will call the super method.
"""
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.rectangle_map_tool.reset()
super(OsmDownloaderDialog, self).reject()
class ConnexionOAPI(object):
"""
Manage connexion to the overpass API
"""
def __init__(self, url="http://overpass-api.de/api/", output=None):
"""
Constructor
@param url:URL of OverPass
@type url:str
@param output:Output desired (XML or JSON)
@type output:str
"""
if not url:
url = "http://overpass-api.de/api/"
self.__url = url
self.data = None
if output not in (None, "json", "xml"):
raise OutPutFormatException
self.__output = output
self.network = QNetworkAccessManager()
self.network_reply = None
self.loop = None
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(
r'output="[a-z]*"', 'output="' + self.__output + '"', query)
query = re.sub(
r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
encoded_query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('data', encoded_query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, self.data):
raise OverpassTimeoutException
else:
return self.data
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass API")
def _end_of_request(self):
self.data = self.network_reply.readAll()
self.loop.quit()
def get_file_from_query(self, query):
"""
Make a query to the overpass and put the result in a temp file
@param query:Query to execute
@type query:str
@return: temporary file path
@rtype: str
"""
query = self.query(query)
tf = tempfile.NamedTemporaryFile(delete=False, suffix=".osm")
tf.write(query)
name_file = tf.name
tf.flush()
tf.close()
return name_file
def get_timestamp(self):
"""
Get the timestamp of the OSM data on the server
@return: Timestamp
@rtype: str
"""
url_query = self.__url + 'timestamp'
try:
return urllib2.urlopen(url=url_query).read()
except urllib2.HTTPError as e:
if e.code == 400:
raise OverpassBadRequestException
def is_valid(self):
"""
Try if the url is valid, NOT TESTED YET
"""
url_query = self.__url + 'interpreter'
try:
urllib2.urlopen(url=url_query)
return True
except urllib2.HTTPError:
return False
class WebClient(BaseWebClient):
"""A webclient with a qtnetwork backend."""
proxy_instance = None
def __init__(self, *args, **kwargs):
"""Initialize this instance."""
super(WebClient, self).__init__(*args, **kwargs)
self.nam = QNetworkAccessManager(QCoreApplication.instance())
self.nam.finished.connect(self._handle_finished)
self.nam.authenticationRequired.connect(self._handle_authentication)
self.nam.proxyAuthenticationRequired.connect(self.handle_proxy_auth)
self.nam.sslErrors.connect(self._handle_ssl_errors)
self.replies = {}
self.proxy_retry = False
self.setup_proxy()
# Force Qt to load the system certificates
QSslSocket.setDefaultCaCertificates(QSslSocket.systemCaCertificates())
# Apply our local certificates as the SSL configuration to be used
# for all QNetworkRequest calls.
self.ssl_config = QSslConfiguration.defaultConfiguration()
ca_certs = self.ssl_config.caCertificates()
try:
for path in glob.glob(
os.path.join(get_cert_dir(), "UbuntuOne*.pem")):
with open(path) as f:
cert = QSslCertificate(f.read())
if cert.isValid():
ca_certs.append(cert)
else:
logger.error("invalid certificate: {}".format(path))
except (IndexError, IOError) as err:
raise WebClientError(
"Unable to configure SSL certificates: {}".format(err))
self.ssl_config.setCaCertificates(ca_certs)
def _set_proxy(self, proxy):
"""Set the proxy to be used."""
QNetworkProxy.setApplicationProxy(proxy)
self.nam.setProxy(proxy)
def setup_proxy(self):
"""Setup the proxy settings if needed."""
# QtNetwork knows how to use the system settings on both Win and Mac
if sys.platform.startswith("linux"):
settings = gsettings.get_proxy_settings()
enabled = len(settings) > 0
if enabled and WebClient.proxy_instance is None:
proxy = build_proxy(settings)
self._set_proxy(proxy)
WebClient.proxy_instance = proxy
elif enabled and WebClient.proxy_instance:
logger.info("Proxy already in use.")
else:
logger.info("Proxy is disabled.")
else:
if WebClient.proxy_instance is None:
logger.info("Querying OS for proxy.")
QNetworkProxyFactory.setUseSystemConfiguration(True)
def handle_proxy_auth(self, proxy, authenticator):
"""Proxy authentication is required."""
logger.info("auth_required %r, %r", self.proxy_username,
proxy.hostName())
if (self.proxy_username is not None
and self.proxy_username != str(authenticator.user())):
authenticator.setUser(self.proxy_username)
WebClient.proxy_instance.setUser(self.proxy_username)
if (self.proxy_password is not None
and self.proxy_password != str(authenticator.password())):
authenticator.setPassword(self.proxy_password)
WebClient.proxy_instance.setPassword(self.proxy_password)
def _perform_request(self, request, method, post_buffer):
"""Return a deferred that will be fired with a Response object."""
d = defer.Deferred()
if method == "GET":
reply = self.nam.get(request)
elif method == "HEAD":
reply = self.nam.head(request)
else:
reply = self.nam.sendCustomRequest(request, method, post_buffer)
self.replies[reply] = d
return d
@defer.inlineCallbacks
def request(self,
iri,
method="GET",
extra_headers=None,
oauth_credentials=None,
post_content=None):
"""Return a deferred that will be fired with a Response object."""
uri = self.iri_to_uri(iri)
request = QNetworkRequest(QUrl(uri))
request.setSslConfiguration(self.ssl_config)
headers = yield self.build_request_headers(uri, method, extra_headers,
oauth_credentials)
for key, value in headers.items():
request.setRawHeader(key, value)
post_buffer = QBuffer()
post_buffer.setData(post_content)
try:
result = yield self._perform_request(request, method, post_buffer)
except ProxyUnauthorizedError as e:
app_proxy = QNetworkProxy.applicationProxy()
proxy_host = app_proxy.hostName() if app_proxy else "proxy server"
got_creds = yield self.request_proxy_auth_credentials(
proxy_host, self.proxy_retry)
if got_creds:
self.proxy_retry = True
result = yield self.request(iri, method, extra_headers,
oauth_credentials, post_content)
else:
excp = WebClientError('Proxy creds needed.', e)
defer.returnValue(excp)
defer.returnValue(result)
def _handle_authentication(self, reply, authenticator):
"""The reply needs authentication."""
if authenticator.user() != self.username:
authenticator.setUser(self.username)
if authenticator.password() != self.password:
authenticator.setPassword(self.password)
def _handle_finished(self, reply):
"""The reply has finished processing."""
assert reply in self.replies
d = self.replies.pop(reply)
error = reply.error()
content = reply.readAll()
if not error:
headers = HeaderDict()
for key, value in reply.rawHeaderPairs():
headers[str(key)].append(str(value))
response = Response(bytes(content), headers)
d.callback(response)
else:
content = unicode(content)
error_string = unicode(reply.errorString())
logger.debug('_handle_finished error (%s,%s).', error,
error_string)
if error == QNetworkReply.AuthenticationRequiredError:
exception = UnauthorizedError(error_string, content)
elif error == QNetworkReply.ProxyAuthenticationRequiredError:
# we are going thru a proxy and we did not auth
exception = ProxyUnauthorizedError(error_string, content)
else:
exception = WebClientError(error_string, content)
d.errback(exception)
def _get_certificate_details(self, cert):
"""Return an string with the details of the certificate."""
detail_titles = {
QSslCertificate.Organization: 'organization',
QSslCertificate.CommonName: 'common_name',
QSslCertificate.LocalityName: 'locality_name',
QSslCertificate.OrganizationalUnitName: 'unit',
QSslCertificate.CountryName: 'country_name',
QSslCertificate.StateOrProvinceName: 'state_name'
}
details = {}
for info, title in detail_titles.items():
details[title] = str(cert.issuerInfo(info))
return self.format_ssl_details(details)
def _get_certificate_host(self, cert):
"""Return the host of the cert."""
return str(cert.issuerInfo(QSslCertificate.CommonName))
def _handle_ssl_errors(self, reply, errors):
"""Handle the case in which we got an ssl error."""
msg = StringIO()
msg.write('SSL errors found; url: %s\n' %
reply.request().url().toString())
for error in errors:
msg.write('========Error=============\n%s (%s)\n' %
(error.errorString(), error.error()))
msg.write('--------Cert Details------\n%s\n' %
self._get_certificate_details(error.certificate()))
msg.write('==========================\n')
logger.error(msg.getvalue())
def force_use_proxy(self, https_settings):
"""Setup this webclient to use the given proxy settings."""
settings = {"https": https_settings}
proxy = build_proxy(settings)
self._set_proxy(proxy)
WebClient.proxy_instance = proxy
def shutdown(self):
"""Shut down all pending requests (if possible)."""
self.nam.deleteLater()
class ConnexionOAPI(object):
"""
Manage connexion to the overpass API
"""
def __init__(self, url="http://overpass-api.de/api/", output=None):
"""
Constructor
@param url:URL of OverPass
@type url:str
@param output:Output desired (XML or JSON)
@type output:str
"""
if not url:
url = "http://overpass-api.de/api/"
self.__url = url
self.data = None
if output not in (None, "json", "xml"):
raise OutPutFormatException
self.__output = output
self.network = QNetworkAccessManager()
self.network_reply = None
self.loop = None
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(r'output="[a-z]*"',
'output="' + self.__output + '"', query)
query = re.sub(r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
encoded_query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('data', encoded_query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, self.data):
raise OverpassTimeoutException
else:
return self.data
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass API")
def _end_of_request(self):
self.data = self.network_reply.readAll()
self.loop.quit()
def get_file_from_query(self, query):
"""
Make a query to the overpass and put the result in a temp file
@param query:Query to execute
@type query:str
@return: temporary file path
@rtype: str
"""
query = self.query(query)
tf = tempfile.NamedTemporaryFile(delete=False, suffix=".osm")
tf.write(query)
name_file = tf.name
tf.flush()
tf.close()
return name_file
def get_timestamp(self):
"""
Get the timestamp of the OSM data on the server
@return: Timestamp
@rtype: str
"""
url_query = self.__url + 'timestamp'
try:
return urllib2.urlopen(url=url_query).read()
except urllib2.HTTPError as e:
if e.code == 400:
raise OverpassBadRequestException
def is_valid(self):
"""
Try if the url is valid, NOT TESTED YET
"""
url_query = self.__url + 'interpreter'
try:
urllib2.urlopen(url=url_query)
return True
except urllib2.HTTPError:
return False
class Browser:
"""
Stateful programmatic web browser class based upon QtWebKit.
>>> browser = Browser()
>>> browser.load("http://www.wordreference.com")
>>> browser.runjs("console.log('I can run Javascript!')")
>>> browser.runjs("_jQuery('div').css('border', 'solid red')") # and jQuery!
>>> browser.select("#esen")
>>> browser.fill("input[name=enit]", "hola")
>>> browser.click("input[name=b]", wait_load=True)
>>> print browser.url, len(browser.html)
>>> browser.close()
"""
ignore_ssl_errors = True
"""@ivar: If True, ignore SSL certificate errors."""
user_agent = None
"""@ivar: User agent for requests (see QWebPage::userAgentForUrl for details)"""
jslib = "jq"
"""@ivar: Library name for jQuery library injected by default to pages."""
download_directory = "."
"""@ivar: Directory where downloaded files will be stored."""
debug_stream = sys.stderr
"""@ivar: File-like stream where debug output will be written."""
debug_level = ERROR
"""@ivar: Debug verbose level (L{ERROR}, L{WARNING}, L{INFO} or L{DEBUG})."""
event_looptime = 0.01
"""@ivar: Event loop dispatcher loop delay (seconds)."""
errorCode = None
errorMessage = None
_javascript_files = ["jquery.min.js", "jquery.simulate.js"]
_javascript_directories = [
os.path.join(os.path.dirname(__file__), "../javascript"),
os.path.join(sys.prefix, "share/spynner/javascript"),
]
def __init__(self, qappargs=None, debug_level=None):
"""
Init a Browser instance.
@param qappargs: Arguments for QApplication constructor.
@param debug_level: Debug level logging (L{ERROR} by default)
"""
self.application = QApplication(qappargs or [])
"""PyQt4.QtGui.Qapplication object."""
if debug_level is not None:
self.debug_level = debug_level
self.webpage = QWebPage()
"""PyQt4.QtWebKit.QWebPage object."""
self.webpage.userAgentForUrl = self._user_agent_for_url
self.webframe = self.webpage.mainFrame()
"""PyQt4.QtWebKit.QWebFrame main webframe object."""
self.webview = None
"""PyQt4.QtWebKit.QWebView object."""
self._url_filter = None
self._html_parser = None
# Javascript
directory = _first(self._javascript_directories, os.path.isdir)
if not directory:
raise SpynnerError("Cannot find javascript directory: %s" %
self._javascript_directories)
self.javascript = "".join(open(os.path.join(directory, fn)).read()
for fn in self._javascript_files)
self.webpage.javaScriptAlert = self._javascript_alert
self.webpage.javaScriptConsoleMessage = self._javascript_console_message
self.webpage.javaScriptConfirm = self._javascript_confirm
self.webpage.javaScriptPrompt = self._javascript_prompt
self._javascript_confirm_callback = None
self._javascript_confirm_prompt = None
# Network Access Manager and cookies
self.manager = QNetworkAccessManager()
"""PyQt4.QtNetwork.QTNetworkAccessManager object."""
self.manager.createRequest = self._manager_create_request
self.webpage.setNetworkAccessManager(self.manager)
self.cookiesjar = _ExtendedNetworkCookieJar()
"""PyQt4.QtNetwork.QNetworkCookieJar object."""
self.manager.setCookieJar(self.cookiesjar)
self.manager.connect(self.manager,
SIGNAL("sslErrors(QNetworkReply *, const QList<QSslError> &)"),
self._on_manager_ssl_errors)
self.manager.connect(self.manager,
SIGNAL('finished(QNetworkReply *)'),
self._on_reply)
self.manager.connect(self.manager,
SIGNAL('authenticationRequired(QNetworkReply *, QAuthenticator *)'),
self._on_authentication_required)
self._operation_names = dict(
(getattr(QNetworkAccessManager, s + "Operation"), s.lower())
for s in ("Get", "Head", "Post", "Put"))
# Webpage slots
self._load_status = None
self._replies = 0
self.webpage.setForwardUnsupportedContent(True)
self.webpage.connect(self.webpage,
SIGNAL('unsupportedContent(QNetworkReply *)'),
self._on_unsupported_content)
self.webpage.connect(self.webpage,
SIGNAL('loadFinished(bool)'),
self._on_load_finished)
self.webpage.connect(self.webpage,
SIGNAL("loadStarted()"),
self._on_load_started)
def _events_loop(self, wait=None):
if wait is None:
wait = self.event_looptime
self.application.processEvents()
time.sleep(wait)
def _on_load_started(self):
self._load_status = None
self._debug(INFO, "Page load started")
def _on_manager_ssl_errors(self, reply, errors):
url = unicode(reply.url().toString())
if self.ignore_ssl_errors:
self._debug(WARNING, "SSL certificate error ignored: %s" % url)
reply.ignoreSslErrors()
else:
self._debug(WARNING, "SSL certificate error: %s" % url)
def _on_authentication_required(self, reply, authenticator):
url = unicode(reply.url().toString())
realm = unicode(authenticator.realm())
self._debug("HTTP auth required: %s (realm: %s)" % (url, realm))
if not self._http_authentication_callback:
self._debug(WARNING, "HTTP auth required, but no callback defined")
return
credentials = self._http_authentication_callback(url, realm)
if credentials:
user, password = credentials
self._debug(INFO, "callback returned HTTP credentials: %s/%s" %
(user, "*"*len(password)))
authenticator.setUser(user)
authenticator.setPassword(password)
else:
self._debug(WARNING, "HTTP auth callback returned no credentials")
def _manager_create_request(self, operation, request, data):
url = unicode(request.url().toString())
operation_name = self._operation_names[operation].upper()
self._debug(INFO, "Request: %s %s" % (operation_name, url))
for h in request.rawHeaderList():
self._debug(DEBUG, " %s: %s" % (h, request.rawHeader(h)))
if self._url_filter:
if self._url_filter(self._operation_names[operation], url) is False:
self._debug(INFO, "URL filtered: %s" % url)
request.setUrl(QUrl("about:blank"))
else:
self._debug(DEBUG, "URL not filtered: %s" % url)
reply = QNetworkAccessManager.createRequest(self.manager,
operation, request, data)
return reply
def _on_reply(self, reply):
self._replies += 1
self._reply_url = unicode(reply.url().toString())
self._reply_status = not bool(reply.error())
if reply.error():
self._debug(WARNING, "Reply error: %s - %d (%s)" %
(self._reply_url, reply.error(), reply.errorString()))
self.errorCode = reply.error()
self.errorMessage = reply.errorString()
else:
self._debug(INFO, "Reply successful: %s" % self._reply_url)
for header in reply.rawHeaderList():
self._debug(DEBUG, " %s: %s" % (header, reply.rawHeader(header)))
def _on_unsupported_content(self, reply, outfd=None):
if not reply.error():
self._start_download(reply, outfd)
else:
self._debug(ERROR, "Error on unsupported content: %s" % reply.errorString())
def _javascript_alert(self, webframe, message):
self._debug(INFO, "Javascript alert: %s" % message)
if self.webview:
QWebPage.javaScriptAlert(self.webpage, webframe, message)
def _javascript_console_message(self, message, line, sourceid):
if line:
self._debug(INFO, "Javascript console (%s:%d): %s" %
(sourceid, line, message))
else:
self._debug(INFO, "Javascript console: %s" % message)
def _javascript_confirm(self, webframe, message):
smessage = unicode(message)
url = webframe.url()
self._debug(INFO, "Javascript confirm (webframe url = %s): %s" %
(url, smessage))
if self._javascript_confirm_callback:
value = self._javascript_confirm_callback(url, smessage)
self._debug(INFO, "Javascript confirm callback returned %s" % value)
return value
return QWebPage.javaScriptConfirm(self.webpage, webframe, message)
def _javascript_prompt(self, webframe, message, defaultvalue, result):
url = webframe.url()
smessage = unicode(message)
self._debug(INFO, "Javascript prompt (webframe url = %s): %s" %
(url, smessage))
if self._javascript_prompt_callback:
value = self._javascript_prompt_callback(url, smessage, defaultvalue)
self._debug(INFO, "Javascript prompt callback returned: %s" % value)
if value in (False, None):
return False
result.clear()
result.append(value)
return True
return QWebPage.javaScriptPrompt(self.webpage, webframe, message,
defaultvalue, result)
def _on_webview_destroyed(self, window):
self.webview = None
def _on_load_finished(self, successful):
self._load_status = successful
status = {True: "successful", False: "error"}[successful]
self._debug(INFO, "Page load finished (%d bytes): %s (%s)" %
(len(self.html), self.url, status))
def _get_filepath_for_url(self, url):
urlinfo = urlparse.urlsplit(url)
path = os.path.join(self.download_directory,
urlinfo.netloc + urlinfo.path)
if not os.path.isdir(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
return path
def _start_download(self, reply, outfd):
def _on_ready_read():
data = reply.readAll()
reply.downloaded_nbytes += len(data)
outfd.write(data)
self._debug(DEBUG, "Read from download stream (%d bytes): %s"
% (len(data), url))
def _on_network_error():
self.debug(ERROR, "Network error on download: %s" % url)
def _on_finished():
self._debug(INFO, "Download finished: %s" % url)
url = unicode(reply.url().toString())
if outfd is None:
path = self._get_filepath_for_url(url)
outfd = open(path, "wb")
reply.connect(reply, SIGNAL("readyRead()"), _on_ready_read)
reply.connect(reply, SIGNAL("NetworkError()"), _on_network_error)
reply.connect(reply, SIGNAL("finished()"), _on_finished)
self._debug(INFO, "Start download: %s" % url)
def _wait_load(self, timeout=None):
self._events_loop(0.0)
if self._load_status is not None:
load_status = self._load_status
self._load_status = None
return load_status
itime = time.time()
while self._load_status is None:
if timeout and time.time() - itime > timeout:
raise SpynnerTimeout("Timeout reached: %d seconds" % timeout)
self._events_loop()
self._events_loop(0.0)
if self._load_status:
jscode = "var %s = jQuery.noConflict();" % self.jslib
self.runjs(self.javascript + jscode, debug=False)
self.webpage.setViewportSize(self.webpage.mainFrame().contentsSize())
load_status = self._load_status
self._load_status = None
return load_status
def _debug(self, level, *args):
if level <= self.debug_level:
kwargs = dict(outfd=self.debug_stream)
_debug(*args, **kwargs)
def _user_agent_for_url(self, url):
if self.user_agent:
return self.user_agent
return QWebPage.userAgentForUrl(self.webpage, url)
def get_js_obj_length(self, res):
if res.type() != res.Map:
return False
resmap = res.toMap()
lenfield = QString(u'length')
if lenfield not in resmap:
return False
return resmap[lenfield].toInt()[0]
def jslen(self, selector):
res = self.runjs("%s('%s')" % (self.jslib, selector))
return self.get_js_obj_length(res)
def _runjs_on_jquery(self, name, code):
res = self.runjs(code)
if self.get_js_obj_length(res) < 1:
raise SpynnerJavascriptError("error on %s: %s" % (name, code))
def _get_html(self):
return unicode(self.webframe.toHtml())
def _get_soup(self):
if not self._html_parser:
raise SpynnerError("Cannot get soup with no HTML parser defined")
return self._html_parser(self.html)
def _get_url(self):
return unicode(self.webframe.url().toString())
# Properties
url = property(_get_url)
"""Current URL."""
html = property(_get_html)
"""Rendered HTML in current page."""
#soup = property(_get_soup)
soup = None #change to none so that changes are retained through mulitple calls
"""HTML soup (see L{set_html_parser})."""
#{ Basic interaction with browser
def load(self, url):
"""Load a web page and return status (a boolean)."""
self.webframe.load(QUrl(url))
return self._wait_load()
def load_request(self, req):
"""Load a network request and return status (a boolean)."""
self.webframe.load(req)
return self._wait_load()
def wait_requests(self, wait_requests = None, url = None, url_regex = None):
if wait_requests:
while self._replies < wait_requests:
self._events_loop()
self._events_loop(0.0)
if url_regex or url:
last_replies = self._replies
while True:
if last_replies != self._replies:
if url_regex:
if re.search(url_regex, self._reply_url):
break
elif url:
if url == self._reply_url:
break
self._events_loop()
self._events_loop(0.0)
def click(self, selector, wait_load=False, wait_requests=None, timeout=None):
"""
Click any clickable element in page.
@param selector: jQuery selector.
@param wait_load: If True, it will wait until a new page is loaded.
@param timeout: Seconds to wait for the page to load before
raising an exception.
@param wait_requests: How many requests to wait before returning. Useful
for AJAX requests.
By default this method will not wait for a page to load.
If you are clicking a link or submit button, you must call this
method with C{wait_load=True} or, alternatively, call
L{wait_load} afterwards. However, the recommended way it to use
L{click_link}.
When a non-HTML file is clicked this method will download it. The
file is automatically saved keeping the original structure (as
wget --recursive does). For example, a file with URL
I{http://server.org/dir1/dir2/file.ext} will be saved to
L{download_directory}/I{server.org/dir1/dir2/file.ext}.
"""
jscode = "%s('%s').simulate('click')" % (self.jslib, selector)
self._replies = 0
self._runjs_on_jquery("click", jscode)
self.wait_requests(wait_requests)
if wait_load:
return self._wait_load(timeout)
def click_link(self, selector, timeout=None):
"""Click a link and wait for the page to load."""
return self.click(selector, wait_load=True, timeout=timeout)
def click_ajax(self, selector, wait_requests=1, timeout=None):
"""Click a AJAX link and wait for the request to finish."""
return self.click(selector, wait_requests=wait_requests, timeout=timeout)
def wait_load(self, timeout=None):
"""
Wait until the page is loaded.
@param timeout: Time to wait (seconds) for the page load to complete.
@return: Boolean state
@raise SpynnerTimeout: If timeout is reached.
"""
return self._wait_load(timeout)
def wait(self, waittime):
"""
Wait some time.
@param waittime: Time to wait (seconds).
This is an active wait, the events loop will be run, so it
may be useful to wait for synchronous Javascript events that
change the DOM.
"""
itime = time.time()
while time.time() - itime < waittime:
self._events_loop()
def close(self):
"""Close Browser instance and release resources."""
if self.webview:
self.destroy_webview()
if self.webpage:
del self.webpage
#}
#{ Webview
def create_webview(self, show=False):
"""Create a QWebView object and insert current QWebPage."""
if self.webview:
raise SpynnerError("Cannot create webview (already initialized)")
self.webview = QWebView()
self.webview.setPage(self.webpage)
window = self.webview.window()
window.setAttribute(Qt.WA_DeleteOnClose)
window.connect(window, SIGNAL('destroyed(QObject *)'),
self._on_webview_destroyed)
if show:
self.show()
def destroy_webview(self):
"""Destroy current QWebView."""
if not self.webview:
raise SpynnerError("Cannot destroy webview (not initialized)")
del self.webview
def show(self):
"""Show webview browser."""
if not self.webview:
raise SpynnerError("Webview is not initialized")
self.webview.show()
def hide(self):
"""Hide webview browser."""
if not self.webview:
raise SpynnerError("Webview is not initialized")
self.webview.hide()
def browse(self):
"""Let the user browse the current page (infinite loop)."""
if not self.webview:
raise SpynnerError("Webview is not initialized")
self.show()
while self.webview:
self._events_loop()
#}
#{ Webframe
def set_webframe_to_default(self):
self.webframe = self.webpage.mainFrame()
def set_webframe(self, framenumber):
cf = self.webframe.childFrames()
try:
self.webframe = cf[int(framenumber)]
except:
raise SpynnerError("childframe does not exist")
"""Inject jquery into frame"""
jscode = "var %s = jQuery.noConflict();" % self.jslib
self.runjs(self.javascript + jscode, debug=False)
#}
#{ Form manipulation
def fill(self, selector, value):
"""Fill an input text with a string value using a jQuery selector."""
escaped_value = value.replace("'", "\\'")
jscode = "%s('%s').val('%s')" % (self.jslib, selector, escaped_value)
self._runjs_on_jquery("fill", jscode)
def check(self, selector):
"""Check an input checkbox using a jQuery selector."""
jscode = "%s('%s').attr('checked', true)" % (self.jslib, selector)
self._runjs_on_jquery("check", jscode)
def uncheck(self, selector):
"""Uncheck input checkbox using a jQuery selector"""
jscode = "%s('%s').attr('checked', false)" % (self.jslib, selector)
self._runjs_on_jquery("uncheck", jscode)
def choose(self, selector, value):
"""Choose a radio input using a jQuery selector."""
escaped_value = value.replace("'", "\\'")
jscode = "%s('%s').filter('[value=%s]').simulate('click')" % (self.jslib, selector, escaped_value)
self._runjs_on_jquery("choose", jscode)
def select(self, selector):
"""Choose a option in a select using a jQuery selector."""
jscode = "%s('%s').attr('selected', 'selected')" % (self.jslib, selector)
self._runjs_on_jquery("select", jscode)
submit = click_link
#}
#{ Javascript
def runjs(self, jscode, debug=True):
"""
Inject Javascript code into the current context of page.
@param jscode: Javascript code to injected.
@param debug: Set to False to disable debug output for this injection.
You can call Jquery even if the original page does not include it
as Spynner injects the library for every loaded page. You must
use C{jq(...)} instead of of C{jQuery} or the common {$(...)}
shortcut.
@note: You can change the jq alias (see L{jslib}).
"""
if debug:
self._debug(DEBUG, "Run Javascript code: %s" % jscode)
#XXX evaluating JS twice must be wrong but finding the bug is proving tricky...
#JavaScriptCore/interpreter/Interpreter.cpp and JavaScriptCore/runtime/Completion.cpp
#JavaScriptCore/runtime/Completion.cpp is catching an exception (sometimes) and
#returning "TypeError: Type error" - BUT it looks like the JS does complete after
#the function has already returned
r = self.webframe.evaluateJavaScript(jscode)
if r.isValid() == False:
r = self.webframe.evaluateJavaScript(jscode)
return r
def set_javascript_confirm_callback(self, callback):
"""
Set function callback for Javascript confirm pop-ups.
By default Javascript confirmations are not answered. If the webpage
you are working pops Javascript confirmations, be sure to set a callback
for them.
Calback signature: C{javascript_confirm_callback(url, message)}
- url: Url where the popup was launched.
- param message: String message.
The callback should return a boolean (True meaning 'yes', False meaning 'no')
"""
self._javascript_confirm_callback = callback
def set_javascript_prompt_callback(self, callback):
"""
Set function callback for Javascript prompt.
By default Javascript prompts are not answered. If the webpage
you are working pops Javascript prompts, be sure to set a callback
for them.
Callback signature: C{javascript_prompt_callback(url, message, defaultvalue)}
- url: Url where the popup prompt was launched.
- message: String message.
- defaultvalue: Default value for prompt answer
The callback should return a string with the answer or None to cancel the prompt.
"""
self._javascript_prompt_callback = callback
#}
#{ Cookies
def get_cookies(self):
"""Return string containing the current cookies in Mozilla format."""
return self.cookiesjar.mozillaCookies()
def set_cookies(self, string_cookies):
"""Set cookies from a string with Mozilla-format cookies."""
return self.cookiesjar.setMozillaCookies(string_cookies)
#}
#{ Proxies
def get_proxy(self):
"""Return string containing the current proxy."""
return self.manager.proxy()
def set_proxy(self, string_proxy):
"""Set proxy [http|socks5]://username:password@hostname:port"""
urlinfo = urlparse.urlparse(string_proxy)
proxy = QNetworkProxy()
if urlinfo.scheme == 'socks5' :
proxy.setType(1)
elif urlinfo.scheme == 'http' :
proxy.setType(3)
else :
proxy.setType(2)
self.manager.setProxy(proxy)
return self.manager.proxy()
proxy.setHostName(urlinfo.hostname)
proxy.setPort(urlinfo.port)
if urlinfo.username != None :
proxy.setUser(urlinfo.username)
else :
proxy.setUser('')
if urlinfo.password != None :
proxy.setPassword(urlinfo.password)
else :
proxy.setPassword('')
self.manager.setProxy(proxy)
return self.manager.proxy()
#}
#{ Download files
def download(self, url, outfd=None):
"""
Download a given URL using current cookies.
@param url: URL or path to download
@param outfd: Output file-like stream. If None, return data string.
@return: Bytes downloaded (None if something went wrong)
@note: If url is a path, the current base URL will be pre-appended.
"""
def _on_reply(reply):
url = unicode(reply.url().toString())
self._download_reply_status = not bool(reply.error())
self._download_reply_status = None
if not urlparse.urlsplit(url).scheme:
url = urlparse.urljoin(self.url, url)
request = QNetworkRequest(QUrl(url))
# Create a new manager to process this download
manager = QNetworkAccessManager()
reply = manager.get(request)
if reply.error():
raise SpynnerError("Download error: %s" % reply.errorString())
reply.downloaded_nbytes = 0
manager.setCookieJar(self.manager.cookieJar())
manager.connect(manager, SIGNAL('finished(QNetworkReply *)'), _on_reply)
outfd_set = bool(outfd)
if not outfd_set:
outfd = StringIO()
self._start_download(reply, outfd)
while self._download_reply_status is None:
self._events_loop()
if outfd_set:
return (reply.downloaded_nbytes if not reply.error() else None)
else:
return outfd.getvalue()
#}
#{ HTML and tag soup parsing
def set_html_parser(self, parser):
"""
Set HTML parser used to generate the HTML L{soup}.
@param parser: Callback called to generate the soup.
When a HTML parser is set for a Browser, the property L{soup} returns
the parsed HTML.
"""
self._html_parser = parser
def html_contains(self, regexp):
"""Return True if current HTML contains a given regular expression."""
return bool(re.search(regexp, self.html))
#}
#{ HTTP Authentication
def set_http_authentication_callback(self, callback):
"""
Set HTTP authentication request callback.
The callback must have this signature:
C{http_authentication_callback(url, realm)}:
- C{url}: URL where the requested was made.
- C{realm}: Realm requiring authentication.
The callback should return a pair of string containing (user, password)
or None if you don't want to answer.
"""
self._http_authentication_callback = callback
#}
#{ Miscellaneous
def snapshot(self, box=None, format=QImage.Format_ARGB32):
"""
Take an image snapshot of the current frame.
@param box: 4-element tuple containing box to capture (x1, y1, x2, y2).
If None, capture the whole page.
@param format: QImage format (see QImage::Format_*).
@return: A QImage image.
Typical usage:
>>> browser.load(url)
>>> browser.snapshot().save("webpage.png")
"""
if box:
x1, y1, x2, y2 = box
w, h = (x2 - x1), (y2 - y1)
image0 = QImage(QSize(x2, y2), format)
painter = QPainter(image0)
self.webpage.mainFrame().render(painter)
painter.end()
image = image0.copy(x1, y1, w, h)
else:
image = QImage(self.webpage.viewportSize(), format)
painter = QPainter(image)
self.webpage.mainFrame().render(painter)
painter.end()
return image
def get_url_from_path(self, path):
"""Return the URL for a given path using the current URL as base."""
return urlparse.urljoin(self.url, path)
def set_url_filter(self, url_filter):
"""
Set function callback to filter URL.
By default all requested elements of a page are loaded. That includes
stylesheets, images and many other elements that you may not need at all.
Use this method to define the callback that will be called every time
a new request is made. The callback must have this signature:
C{my_url_filter(operation, url)}:
- C{operation}: string with HTTP operation: C{get}, C{head},
C{post} or C{put}.
- C{url}: requested item URL.
It should return C{True} (proceed) or C{False} (reject).
"""
self._url_filter = url_filter
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = "http://osm.linfiniti.com/buildings-shp"
self.roads_url = "http://osm.linfiniti.com/roads-shp"
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr("InaSAFE OpenStreetMap Downloader")
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if proxy is not None:
self.network_manager.setProxy(proxy)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent will be used to determine the area for which '
'you want data to be retrieved. You can adjust it manually using '
'the bounding box options below.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self):
""" Update extent value in GUI based from value in map."""
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
def validate_extent(self):
"""Validate the bounding box before user click OK to download.
:return: True if the bounding box is valid, otherwise False
:rtype: bool
"""
min_latitude = float(str(self.min_latitude.text()))
max_latitude = float(str(self.max_latitude.text()))
min_longitude = float(str(self.min_longitude.text()))
max_longitude = float(str(self.max_longitude.text()))
# min_latitude < max_latitude
if min_latitude >= max_latitude:
return False
# min_longitude < max_longitude
if min_longitude >= max_longitude:
return False
# -90 <= latitude <= 90
if min_latitude < -90 or min_latitude > 90:
return False
if max_latitude < -90 or max_latitude > 90:
return False
# -180 <= longitude <= 180
if min_longitude < -180 or min_longitude > 180:
return False
if max_longitude < -180 or max_longitude > 180:
return False
return True
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Validate extent
valid_flag = self.validate_extent()
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(self, error_dialog_title, message)
return
# Get all the feature types
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads']
elif index == 1:
feature_types = ['buildings']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
self.download(feature_type)
self.load_shapefile(feature_type)
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(self, error_dialog_title, str(exception))
self.progress_dialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = str(self.output_directory.text())
if os.path.exists(path):
return
title = self.tr("Directory %s not exist") % path
question = self.tr(
"Directory %s not exist. Do you want to create it?") % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type):
"""Download shapefiles from Linfiniti server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError, CanceledImportDialogError
"""
# preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = (
'{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(
min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude
)
output_prefix = self.filename_prefix.text()
if feature_type == 'buildings':
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.buildings_url, box=box)
else:
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.roads_url, box=box)
if output_prefix is not None:
url += '&output_prefix=%s' % output_prefix
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path)
self.extract_zip(path, str(self.output_directory.text()))
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
self.progress_dialog.setMaximum(100)
self.progress_dialog.setValue(0)
label_text = self.tr("Downloading shapefile")
self.progress_dialog.setLabelText(label_text)
# Download Process
downloader = FileDownloader(
self.network_manager, url, output_path, self.progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
raise DownloadError(error_message)
@staticmethod
def extract_zip(path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
handle = open(path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
output_path = os.path.join(output_dir, name)
output_file = open(output_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when buildings.shp not exist
"""
output_prefix = self.filename_prefix.text()
path = str(self.output_directory.text())
path = os.path.join(path, '%s%s.shp' % (output_prefix, feature_type))
if not os.path.exists(path):
message = self.tr(
"%s don't exist. The server doesn't have any data.")
raise ImportDialogError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="http://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QNetworkAccessManager()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('q', query)
url_query.addQueryItem('info', 'QgisOSMDataPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "OSMData")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = 'http://osm.linfiniti.com/buildings-shp'
self.building_points_url = \
'http://osm.linfiniti.com/building-points-shp'
self.roads_url = 'http://osm.linfiniti.com/roads-shp'
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('InaSAFE OpenStreetMap Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if proxy is not None:
self.network_manager.setProxy(proxy)
self.restore_state()
# Setup the rectangle map tool
self.canvas = iface.mapCanvas()
self.rectangle_map_tool = \
RectangleMapTool(self.canvas)
self.rectangle_map_tool.rectangle_created.connect(
self.update_extent_from_rectangle)
self.button_extent_rectangle.clicked.connect(
self.drag_rectangle_on_map_canvas)
# Setup pan tool
self.pan_tool = QgsMapToolPan(self.canvas)
self.canvas.setMapTool(self.pan_tool)
self.update_extent_from_map_canvas()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent, when opening this window, will be used to '
'determine the area for which you want data to be retrieved.'
'You can interactively select the area by using the '
'\'select on map\' button - which will temporarily hide this '
'window and allow you to drag a rectangle on the map. After you '
'have finished dragging the rectangle, this window will '
'reappear.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self, extent):
"""Update extent value in GUI based from an extent.
:param extent: A list in the form [xmin, ymin, xmax, ymax] where all
coordinates provided are in Geographic / EPSG:4326.
:type extent: list
"""
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
def update_extent_from_map_canvas(self):
"""Update extent value in GUI based from value in map.
.. note:: Delegates to update_extent()
"""
self.groupBox.setTitle(self.tr('Bounding box from the map canvas'))
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.update_extent(extent)
def update_extent_from_rectangle(self):
"""Update extent value in GUI based from the QgsMapTool rectangle.
.. note:: Delegates to update_extent()
"""
self.show()
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.canvas.setMapTool(self.pan_tool)
rectangle = self.rectangle_map_tool.rectangle()
if rectangle:
self.groupBox.setTitle(self.tr('Bounding box from rectangle'))
extent = rectangle_geo_array(rectangle, self.iface.mapCanvas())
self.update_extent(extent)
def validate_extent(self):
"""Validate the bounding box before user click OK to download.
:return: True if the bounding box is valid, otherwise False
:rtype: bool
"""
min_latitude = float(str(self.min_latitude.text()))
max_latitude = float(str(self.max_latitude.text()))
min_longitude = float(str(self.min_longitude.text()))
max_longitude = float(str(self.max_longitude.text()))
# min_latitude < max_latitude
if min_latitude >= max_latitude:
return False
# min_longitude < max_longitude
if min_longitude >= max_longitude:
return False
# -90 <= latitude <= 90
if min_latitude < -90 or min_latitude > 90:
return False
if max_latitude < -90 or max_latitude > 90:
return False
# -180 <= longitude <= 180
if min_longitude < -180 or min_longitude > 180:
return False
if max_longitude < -180 or max_longitude > 180:
return False
return True
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory."""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def drag_rectangle_on_map_canvas(self):
"""Hide the dialog and allow the user to draw a rectangle."""
self.hide()
self.rectangle_map_tool.reset()
self.canvas.unsetMapTool(self.pan_tool)
self.canvas.setMapTool(self.rectangle_map_tool)
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Lock the groupbox
self.groupBox.setDisabled(True)
# Validate extent
valid_flag = self.validate_extent()
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the groupbox
self.groupBox.setEnabled(True)
return
# Get all the feature types
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads', 'building-points']
elif index == 1:
feature_types = ['buildings']
elif index == 2:
feature_types = ['building-points']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_checkBox.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, feature_type, overwrite)
self.download(feature_type, output_base_file_path)
try:
self.load_shapefile(feature_type, output_base_file_path)
except FileMissingError as exception:
display_warning_message_box(
self,
error_dialog_title,
exception.message)
self.done(QDialog.Accepted)
self.rectangle_map_tool.reset()
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, error_dialog_title, exception.message)
self.progress_dialog.cancel()
finally:
# Unlock the groupbox
self.groupBox.setEnabled(True)
def get_output_base_path(
self,
output_directory,
output_prefix,
feature_type,
overwrite):
"""Get a full base name path to save the shapefile.
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
path = os.path.join(
output_directory, '%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix(
'%s.shp' % path, separator)
if suffix:
path = os.path.join(output_directory, '%s%s%s%s' % (
output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return OsmDownloaderDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type, output_base_path):
"""Download shapefiles from Kartoza server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param output_base_path: The base path of the shape file.
:type output_base_path: str
:raises: ImportDialogError, CanceledImportDialogError
"""
# preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = (
'{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(
min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude
)
if feature_type == 'buildings':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.buildings_url, box=box)
elif feature_type == 'building-points':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.building_points_url, box=box)
else:
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.roads_url, box=box)
if 'LANG' in os.environ:
env_lang = os.environ['LANG']
url += '&lang=%s' % env_lang
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path, feature_type)
self.extract_zip(path, output_base_path)
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path, feature_type):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
# Infinite progress bar when the server is fetching data.
# The progress bar will be updated with the file size later.
self.progress_dialog.setMaximum(0)
self.progress_dialog.setMinimum(0)
self.progress_dialog.setValue(0)
# Get a pretty label from feature_type, but not translatable
label_feature_type = feature_type.replace('-', ' ')
label_text = self.tr('Fetching %s' % label_feature_type)
self.progress_dialog.setLabelText(label_text)
# Download Process
downloader = FileDownloader(
self.network_manager, url, output_path, self.progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
if result[0] == QNetworkReply.OperationCanceledError:
raise CanceledImportDialogError(error_message)
else:
raise DownloadError(error_message)
@staticmethod
def extract_zip(zip_path, destination_base_path):
"""Extract different extensions to the destination base path.
Example : test.zip contains a.shp, a.dbf, a.prj
and destination_base_path = '/tmp/CT-buildings
Expected result :
- /tmp/CT-buildings.shp
- /tmp/CT-buildings.dbf
- /tmp/CT-buildings.prj
If two files in the zip with the same extension, only one will be
copied.
:param zip_path: The path of the .zip file
:type zip_path: str
:param destination_base_path: The destination base path where the shp
will be written to.
:type destination_base_path: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
handle = open(zip_path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
extension = os.path.splitext(name)[1]
output_final_path = u'%s%s' % (destination_base_path, extension)
output_file = open(output_final_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers')
)
def reject(self):
"""Redefinition of the reject() method
to remove the rectangle selection tool.
It will call the super method.
"""
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.rectangle_map_tool.reset()
super(OsmDownloaderDialog, self).reject()
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="http://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QNetworkAccessManager()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('q', query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
socket.disconnectFromHost()
if proxy_socket.error() != QTcpSocket.RemoteHostClosedError:
url = proxy_socket.property('url').toUrl()
error_string = proxy_socket.errorString()
if self.debug:
self.log.write('Error for %s %s\n\n' % (url, error_string))
proxy_socket.deleteLater()
if __name__ == '__main__':
app = QCoreApplication(sys.argv)
server = HTTPProxy()
manager = QNetworkAccessManager()
manager.finished.connect(server.stopServing)
manager.finished.connect(app.quit)
proxy = QNetworkProxy()
proxy.setType(QNetworkProxy.HttpProxy)
proxy.setHostName('127.0.0.1')
proxy.setPort(server.port())
proxy.setUser(server.username)
proxy.setPassword(server.password)
manager.setProxy(proxy)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
sys.exit(app.exec_())
