def ProgressBar(n_steps, label="", allow_cancel=False, parent=None):
"""
Progressbar context manager for showing progress of workflow to user. Example::
with emzed.gui.ProgressBar(n_steps=100, allow_cancel=True) as handler:
for i in range(100):
# we simulate work of step i
# we update progressbar
handler.update(i, "step %03d" % i)
# we can check if user pressed "Cancel" button and stop our "workflow":
if handler.is_canceled():
break
"""
app = guidata.qapplication()
dlg = QProgressDialog(parent)
dlg.setLabelText(label)
dlg.setAutoClose(False)
dlg.setAutoReset(False)
if allow_cancel:
dlg.setCancelButtonText("Cancel")
dlg.setMaximum(n_steps)
class ProgressBarHandler(object):
def __init__(self, n_steps, dlg):
self._dlg = dlg
self._n_steps = n_steps
self._n = 0
self._canceled = False
dlg.canceled.connect(self._set_canceled)
dlg.setValue(0)
def _set_canceled(self):
self._canceled = True
dlg.close()
def update(self, n, message=None):
app.processEvents()
self._n = n
dlg.setValue(n + 1)
if message is not None:
dlg.setLabelText(message)
dlg.update()
app.processEvents()
def is_canceled(self):
return self._canceled
dlg.activateWindow()
dlg.show()
dlg.raise_()
app.processEvents()
handler = ProgressBarHandler(n_steps, dlg)
yield handler
dlg.close()
def dump_script(self):
to_dump = self.get_checked([self.ui.treeFileList.topLevelItem(i) for i in range(self.ui.treeFileList.topLevelItemCount())])
if not to_dump:
QtGui.QMessageBox.warning(self, "No Selection", "No folders have beens selected to dump.")
return
out_file = get_save_file(self, self.last_file, "Text files (*.txt)")
if out_file == "":
return
translated = not self.ui.chkUntranslated.isChecked()
strip_clt = self.ui.chkStripClt.isChecked()
only_voiced = self.ui.chkOnlyVoiced.isChecked()
line_numbers = self.ui.chkLineNumbers.isChecked()
progress = QProgressDialog("Dumping...", QtCore.QString(), 0, len(to_dump), self)
progress.setWindowTitle("Dumping...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
# print out_file
with open(out_file, "wb") as f:
for dir in to_dump:
progress.setLabelText("Dumping %s..." % dir)
f.write(script_to_text(dir, translated, strip_clt, only_voiced, line_numbers).encode("UTF-8"))
progress.setValue(progress.value() + 1)
progress.close()
self.last_file = out_file
def create_progress_bar(self, title):
progress = QProgressDialog(self)
progress.setAutoClose(False)
progress.setWindowTitle(title)
progress.setCancelButton(None)
progress.setWindowModality(Qt.WindowModal)
return progress
def createProgress(self, title):
progress = QProgressDialog(title, "Abort", 0, 10)
progress.setAttribute(Qt.WA_DeleteOnClose)
progress.setAutoClose(True)
progress.setMinimumDuration(500)
progress.setValue(0)
return progress
def create_progress_bar(self, title):
progress = QProgressDialog(self)
progress.setAutoClose(False)
progress.setWindowTitle(title)
progress.setCancelButton(None)
progress.setWindowModality(Qt.WindowModal)
return progress
def createProgress(self, title):
progress = QProgressDialog(title, "Abort", 0, 10)
progress.setAttribute(Qt.WA_DeleteOnClose)
progress.setAutoClose(True)
progress.setMinimumDuration(500)
progress.setValue(0)
return progress
class IsoBuilder():
def __init__(self, parent=None):
self.parent = parent
self.process = None
def __parse_output(self):
if not self.process:
return
output = QString(self.process.readAll())
output = output.split("\n", QString.SkipEmptyParts)
for line in output:
line = common.qt_to_unicode(line)
match = OUTPUT_RE.match(line)
if match == None:
continue
percent = float(match.group(1))
self.progress.setValue(percent)
def build_iso(self, directory, iso_file):
if self.process:
return
directory = os.path.abspath(directory)
iso_file = os.path.abspath(iso_file)
self.progress = QProgressDialog("Building ISO...", QtCore.QString(), 0,
0, self.parent)
self.progress.setWindowTitle("Building ISO")
self.progress.setWindowModality(Qt.Qt.WindowModal)
self.progress.setAutoClose(False)
self.progress.setMinimumDuration(1000)
self.progress.show()
self.progress.setValue(0)
self.progress.setMaximum(100)
self.process = QProcess()
self.process.finished.connect(self.__build_finished)
self.process.setReadChannel(QProcess.StandardError)
self.process.readyRead.connect(self.__parse_output)
self.process.start("tools/mkisofs", [
"-sort", "data/file_order.txt", "-iso-level", "4", "-xa", "-A",
"PSP GAME", "-V", "DANGANRONPA", "-sysid", "PSP GAME", "-volset",
"DANGANRONPA", "-p", "SPIKE", "-publisher", "SPIKE", "-o",
iso_file, directory
])
def __build_finished(self, code, status):
self.progress.close()
self.process = None
class IsoBuilder():
def __init__(self, parent = None):
self.parent = parent
self.process = None
def __parse_output(self):
if not self.process:
return
output = QString(self.process.readAll())
output = output.split("\n", QString.SkipEmptyParts)
for line in output:
line = common.qt_to_unicode(line)
match = OUTPUT_RE.match(line)
if match == None:
continue
percent = float(match.group(1))
self.progress.setValue(percent)
def build_iso(self, directory, iso_file):
if self.process:
return
directory = os.path.abspath(directory)
iso_file = os.path.abspath(iso_file)
self.progress = QProgressDialog("Building ISO...", QtCore.QString(), 0, 0, self.parent)
self.progress.setWindowTitle("Building ISO")
self.progress.setWindowModality(Qt.Qt.WindowModal)
self.progress.setAutoClose(False)
self.progress.setMinimumDuration(1000)
self.progress.show()
self.progress.setValue(0)
self.progress.setMaximum(100)
self.process = QProcess()
self.process.finished.connect(self.__build_finished)
self.process.setReadChannel(QProcess.StandardError)
self.process.readyRead.connect(self.__parse_output)
self.process.start("tools/mkisofs", ["-sort", "data/file_order.txt", "-iso-level", "4", "-xa", "-A", "PSP GAME", "-V", "DANGANRONPA", "-sysid", "PSP GAME", "-volset", "DANGANRONPA", "-p", "SPIKE", "-publisher", "SPIKE", "-o", iso_file, directory])
def __build_finished(self, code, status):
self.progress.close()
self.process = None
def showAllChartWindows(sg, maximized=False):
dlg = QProgressDialog(SimuVis4.Globals.mainWin)
dlg.setWindowModality(Qt.WindowModal)
dlg.setMaximum(len(sg.charts))
dlg.setAutoClose(True)
dlg.setMinimumDuration(0)
dlg.setValue(0)
i = 1
app = SimuVis4.Globals.application
for name, chart in sg.charts.items():
dlg.setLabelText(name)
app.processEvents()
showChartWindow(chart, maximized)
dlg.setValue(i)
i += 1
if dlg.wasCanceled():
break
def generate_font(self):
progress = QProgressDialog("", QtCore.QString(), 0, 0, self)
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setWindowTitle("Generating font...")
progress.setLabelText("Generating font...")
progress.setMinimumDuration(0)
progress.setAutoClose(False)
# Thread this because it's slow as hell and we don't want to lock up the GUI.
thread = threading.Thread(target=self.__generate_font__)
thread.start()
while thread.isAlive():
thread.join(THREAD_TIMEOUT)
# It has to change by some amount or it won't update and the UI will lock up.
progress.setValue(progress.value() - 1)
progress.close()
def progress_dialog(progress=0, title='CQFS Progress', label='Running...'):
dialog = QProgressDialog()
dialog.setWindowTitle(title)
dialog.setLabelText(label)
dialog.setAutoClose(True)
bar = QProgressBar(dialog)
bar.setTextVisible(True)
bar.setValue(progress)
bar.setMaximum(100)
dialog.setBar(bar)
dialog.setMinimumWidth(300)
dialog.show()
if int(progress) == 0:
bar.setValue(0)
return dialog, bar
def generate_font(self):
progress = QProgressDialog("", QtCore.QString(), 0, 0, self)
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setWindowTitle("Generating font...")
progress.setLabelText("Generating font...")
progress.setMinimumDuration(0)
progress.setAutoClose(False)
# Thread this because it's slow as hell and we don't want to lock up the GUI.
thread = threading.Thread(target = self.__generate_font__)
thread.start()
while thread.isAlive():
thread.join(THREAD_TIMEOUT)
# It has to change by some amount or it won't update and the UI will lock up.
progress.setValue(progress.value() - 1)
progress.close()
def dump_script(self):
to_dump = self.get_checked([
self.ui.treeFileList.topLevelItem(i)
for i in range(self.ui.treeFileList.topLevelItemCount())
])
if not to_dump:
QtGui.QMessageBox.warning(
self, "No Selection",
"No folders have beens selected to dump.")
return
out_file = get_save_file(self, self.last_file, "Text files (*.txt)")
if out_file == "":
return
translated = not self.ui.chkUntranslated.isChecked()
strip_clt = self.ui.chkStripClt.isChecked()
only_voiced = self.ui.chkOnlyVoiced.isChecked()
line_numbers = self.ui.chkLineNumbers.isChecked()
progress = QProgressDialog("Dumping...", QtCore.QString(), 0,
len(to_dump), self)
progress.setWindowTitle("Dumping...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
# print out_file
with open(out_file, "wb") as f:
for dir in to_dump:
progress.setLabelText("Dumping %s..." % dir)
f.write(
script_to_text(dir, translated, strip_clt, only_voiced,
line_numbers).encode("UTF-8"))
progress.setValue(progress.value() + 1)
progress.close()
self.last_file = out_file
def saveAllChartImages(sg):
if SimuVis4.Globals.config.getboolean('matplot', 'set_default_backend'):
QMessageBox.warning(SimuVis4.Globals.mainWin,
QCoreApplication.translate('DataStorageBrowser', 'Configuration error'),
QCoreApplication.translate('DataStorageBrowser',
"""The option "set_default_backend" in section "matplot" is enabled.
The requested action will not work with this setting.
Change this setting and restart the application to make this work!"""))
return
f = QFileDialog.getExistingDirectory(SimuVis4.Globals.mainWin,
QCoreApplication.translate('DataStorageBrowser',
"Select a folder (existing image files will be overwritten!)"),
SimuVis4.Globals.defaultFolder)
if f.isEmpty():
return
folder = unicode(f)
SimuVis4.Globals.defaultFolder = folder
dlg = QProgressDialog(SimuVis4.Globals.mainWin)
dlg.setWindowModality(Qt.WindowModal)
dlg.setMaximum(len(sg.charts))
dlg.setAutoClose(True)
dlg.setMinimumDuration(0)
dlg.setValue(0)
i = 1
app = SimuVis4.Globals.application
for name, chart in sg.charts.items():
fileName = "%s.png" % name
dlg.setLabelText(fileName)
app.processEvents()
#chart.setTimeslice(1*86400)
chart(starttime=(sg.stop-chart.standardSlice), filename=os.path.join(folder, fileName))
dlg.setValue(i)
i += 1
if dlg.wasCanceled():
break
class CpkPacker():
def __init__(self, parent = None):
self.parent = parent
self.process = None
def __pack_cpk(self, csv, cpk):
self.progress.setValue(0)
self.progress.setMaximum(100000)
self.progress.setLabelText("Building %s" % cpk)
process = QProcess()
process.start("tools/cpkmakec", [csv, cpk, "-align=2048", "-mode=FILENAME"])
percent = 0
while not process.waitForFinished(100):
output = QString(process.readAll())
output = output.split("\n", QString.SkipEmptyParts)
for line in output:
line = common.qt_to_unicode(line)
match = OUTPUT_RE.search(line)
if match == None:
continue
percent = float(match.group(1)) * 1000
self.progress.setValue(percent)
percent += 1
def __cache_outdated(self, src_dir, cache_file):
if not os.path.isfile(cache_file):
return True
cache_updated = os.path.getmtime(cache_file)
for src_file in list_all_files(src_dir):
if os.path.getmtime(src_file) > cache_updated:
return True
return False
def create_archives(self):
try:
self.width = self.parent.width()
self.height = self.parent.height()
self.x = self.parent.x()
self.y = self.parent.y()
except:
self.width = 1920
self.height = 1080
self.x = 0
self.y = 0
self.progress = QProgressDialog("Reading...", QtCore.QString(), 0, 7600, self.parent)
self.progress.setWindowModality(Qt.Qt.WindowModal)
self.progress.setValue(0)
self.progress.setAutoClose(False)
self.progress.setMinimumDuration(0)
USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "USRDIR")
eboot_path = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN")
eboot = BitStream(filename = eboot_path)
eboot = eboot_patch.apply_eboot_patches(eboot)
# So we can loop. :)
ARCHIVE_INFO = [
{
"dir": common.editor_config.data00_dir,
"cpk": os.path.join(USRDIR, "data00.cpk"),
"csv": os.path.join("data", "data00.csv" if not common.editor_config.quick_build else "data00-quick.csv"),
"name": "data00.cpk",
"pack": common.editor_config.pack_data00,
},
{
"dir": common.editor_config.data01_dir,
"cpk": os.path.join(USRDIR, "data01.cpk"),
"csv": os.path.join("data", "data01.csv" if not common.editor_config.quick_build else "data01-quick.csv"),
"name": "data01.cpk",
"pack": common.editor_config.pack_data01,
},
]
# temp_dir = tempfile.mkdtemp(prefix = "sdse-")
temp_dir = common.editor_config.build_cache
for archive in ARCHIVE_INFO:
if not archive["pack"]:
continue
self.progress.setWindowTitle("Building " + archive["name"])
csv_template_f = open(archive["csv"], "rb")
csv_template = csv.reader(csv_template_f)
csv_out_path = os.path.join(temp_dir, "cpk.csv")
csv_out_f = open(csv_out_path, "wb")
csv_out = csv.writer(csv_out_f)
for row in csv_template:
if len(row) < 4:
continue
base_path = row[0]
real_path = os.path.join(archive["dir"], base_path)
out_path = os.path.join(temp_dir, archive["name"], base_path)
self.progress.setValue(self.progress.value() + 1)
self.progress.setLabelText("Reading...\n%s" % real_path)
# All items in the CPK list should be files.
# Therefore, if we have a directory, then it needs to be packed.
if os.path.isdir(real_path):
if self.__cache_outdated(real_path, out_path):
out_dir = os.path.dirname(out_path)
try:
os.makedirs(out_dir)
except:
pass
data = pack_dir(real_path)
with open(out_path, "wb") as out_file:
data.tofile(out_file)
del data
elif os.path.isfile(real_path):
# If it's a file, though, we can just use it directly.
out_path = real_path
row[0] = out_path
csv_out.writerow(row)
csv_template_f.close()
csv_out_f.close()
self.__pack_cpk(csv_out_path, archive["cpk"])
self.progress.setWindowTitle("Building...")
self.progress.setLabelText("Saving EBOOT.BIN...")
self.progress.setValue(self.progress.maximum())
with open(eboot_path, "wb") as f:
eboot.tofile(f)
# self.progress.setLabelText("Deleting temporary files...")
# shutil.rmtree(temp_dir)
self.progress.close()
class SymbologySharingDialog(QtGui.QDialog, FORM_CLASS):
TAB_ALL = 0
TAB_INSTALLED = 1
TAB_SETTINGS = 2
def __init__(self, parent=None, iface=None):
"""Constructor.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
super(SymbologySharingDialog, self).__init__(parent)
self.setupUi(self)
self.iface = iface
self.repository_manager = RepositoryManager()
# Init the message bar
self.message_bar = QgsMessageBar(self)
self.message_bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.vlayoutRightColumn.insertWidget(0, self.message_bar)
# Mock plugin manager dialog
self.resize(796, 594)
self.setMinimumSize(QSize(790, 0))
self.setModal(True)
self.button_edit.setEnabled(False)
self.button_delete.setEnabled(False)
# Set QListWidgetItem
# All
icon_all = QIcon()
icon_all.addFile(
resources_path('img', 'plugin.svg'),
QSize(),
QIcon.Normal,
QIcon.Off)
item_all = QListWidgetItem()
item_all.setIcon(icon_all)
item_all.setText(self.tr('All'))
# Installed
icon_installed = QIcon()
icon_installed.addFile(
resources_path('img', 'plugin-installed.svg'),
QSize(),
QIcon.Normal,
QIcon.Off)
item_installed = QListWidgetItem()
item_installed.setIcon(icon_installed)
item_installed.setText(self.tr('Installed'))
item_all.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
# Settings
icon_settings = QIcon()
icon_settings.addFile(
resources_path('img', 'settings.svg'),
QSize(),
QIcon.Normal,
QIcon.Off)
item_settings = QListWidgetItem()
item_settings.setIcon(icon_settings)
item_settings.setText(self.tr('Settings'))
item_all.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
# Add the list widget item to the widget
self.menu_list_widget.addItem(item_all)
self.menu_list_widget.addItem(item_installed)
self.menu_list_widget.addItem(item_settings)
# Slots
self.button_add.clicked.connect(self.add_repository)
self.button_edit.clicked.connect(self.edit_repository)
self.button_delete.clicked.connect(self.delete_repository)
self.menu_list_widget.currentRowChanged.connect(self.set_current_tab)
# Creating progress dialog for downloading stuffs
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('Symbology Sharing')
self.progress_dialog.setWindowTitle(title)
# Populate tree repositories with registered repositories
self.populate_tree_repositories()
def set_current_tab(self, index):
"""Set stacked widget based on active tab.
:param index: The index of the active list widget item.
:type index: int
"""
if index == (self.menu_list_widget.count() - 1):
# Switch to settings tab
self.stacked_menu_widget.setCurrentIndex(1)
else:
# Switch to plugins tab
self.stacked_menu_widget.setCurrentIndex(0)
def add_repository(self):
"""Open add repository dialog."""
dlg = ManageRepositoryDialog(self)
if not dlg.exec_():
return
for repo in self.repository_manager.repositories.values():
if dlg.line_edit_url.text().strip() == repo['url']:
self.message_bar.pushMessage(
self.tr(
'Unable to add another repository with the same URL!'),
QgsMessageBar.WARNING, 5)
return
repo_name = dlg.line_edit_name.text()
repo_url = dlg.line_edit_url.text().strip()
if repo_name in self.repository_manager.repositories:
repo_name += '(2)'
settings = QSettings()
settings.beginGroup(repo_settings_group())
settings.setValue(repo_name + '/url', repo_url)
# Fetch metadata
#TODO: Wrap RemoteRepository class into RepositoryManager
# This dialod will only need to call RepositoryManager.
# RepositoryManager will take care of the rest
remote_repository = RemoteRepository(repo_url)
remote_repository.fetch_metadata(self.progress_dialog)
# Show metadata
#TODO: Process this instead of showing it on message box :)
QMessageBox.information(
self,
self.tr("Test"),
remote_repository.metadata.data())
# Refresh tree repository
self.refresh_tree_repositories()
self.set_enabled_edit_delete_button(False)
def edit_repository(self):
"""Open edit repository dialog."""
selected_item = self.tree_repositories.currentItem()
if selected_item:
repo_name = selected_item.text(0)
if not repo_name:
return
dlg = ManageRepositoryDialog(self)
dlg.line_edit_name.setText(repo_name)
dlg.line_edit_url.setText(
self.repository_manager.repositories[repo_name]['url'])
if not dlg.exec_():
return
# Check if the changed URL is already there in the repo
new_url = dlg.line_edit_url.text().strip()
old_url = self.repository_manager.repositories[repo_name]['url']
for repo in self.repository_manager.repositories.values():
if new_url == repo['url'] and (old_url != new_url):
self.message_bar.pushMessage(
self.tr(
'Unable to add another repository with the same URL!'),
QgsMessageBar.WARNING, 5)
return
# Delete old repo and create a new entry
settings = QSettings()
settings.beginGroup(repo_settings_group())
settings.remove(repo_name)
new_name = dlg.line_edit_name.text()
if new_name in self.repository_manager.repositories and new_name != repo_name:
new_name += '(2)'
settings.setValue(new_name + '/url', new_url)
# Refresh tree repository
self.refresh_tree_repositories()
self.set_enabled_edit_delete_button(False)
def delete_repository(self):
"""Delete a repository in the tree widget."""
selected_item = self.tree_repositories.currentItem()
if selected_item:
repo_name = selected_item.text(0)
if not repo_name:
return
# Check if it's the official repository
settings = QSettings()
settings.beginGroup(repo_settings_group())
if settings.value(repo_name + '/url') == self.repository_manager.OFFICIAL_REPO[1]:
self.message_bar.pushMessage(
self.tr(
'You can not remove the official repository!'),
QgsMessageBar.WARNING, 5)
return
warning = self.tr('Are you sure you want to remove the following '
'repository?') + '\n' + repo_name
if QMessageBox.warning(
self,
self.tr("QGIS Symbology Sharing"),
warning,
QMessageBox.Yes,
QMessageBox.No) == QMessageBox.No:
return
settings.remove(repo_name)
# Refresh tree repository
self.refresh_tree_repositories()
self.set_enabled_edit_delete_button(False)
def refresh_tree_repositories(self):
"""Refresh tree repositories using new repositories data."""
self.repository_manager.load()
self.populate_tree_repositories()
def populate_tree_repositories(self):
"""Populate dictionary repositories to the tree widget."""
# Clear the current tree widget
self.tree_repositories.clear()
# Export the updated ones from the repository manager
for repo_name in self.repository_manager.repositories:
url = self.repository_manager.repositories[repo_name]['url']
item = QTreeWidgetItem(self.tree_repositories)
item.setText(0, repo_name)
item.setText(1, url)
self.tree_repositories.resizeColumnToContents(0)
self.tree_repositories.resizeColumnToContents(1)
self.tree_repositories.sortItems(1, Qt.AscendingOrder)
def on_tree_repositories_itemSelectionChanged(self):
"""Slot for when the itemSelectionChanged signal emitted."""
# Activate edit and delete button
self.set_enabled_edit_delete_button(True)
def set_enabled_edit_delete_button(self, is_enabled):
"""Disable edit and delete button.
:param is_enabled: Boolean is enabled or not.
:type is_enabled: bool
"""
self.button_edit.setEnabled(is_enabled)
self.button_delete.setEnabled(is_enabled)
class specread:
def __init__(self,
specfile,
startLineNum=0,
endScanNum=0,
beamline='APS-15IDC',
det='CCD',
data={},
par={}):
self.Data = data
self.Par = par
self.specfile = specfile
if beamline == 'APS-15IDC':
self.APS_15IDC(startLineNum=startLineNum,
endScanNum=endScanNum,
det=det)
if beamline == 'APS-9IDC':
self.APS_9IDC(startLineNum=startLineNum,
endScanNum=endScanNum,
det=det)
def updateProgress(self):
self.progressDialog.setValue(self.progressDialog.value() + 1)
def APS_15IDC(self, startLineNum=0, endScanNum=0, det='CCD'):
"""
Function to read a complete spec File collected at APS 15IDC
"""
self.progressDialog = QProgressDialog('Reading scans form SPEC File:',
'Abort', 0, 100)
self.progressDialog.setWindowModality(Qt.WindowModal)
self.progressDialog.setWindowTitle('Wait')
self.progressDialog.setAutoClose(True)
self.progressDialog.setAutoReset(True)
self.progressDialog.setMinimum(1)
self.Data['YCol'] = 'Apex2'
self.Data['NCol'] = 'Monc'
fid = open(self.specfile)
fdata = fid.readlines()
self.SpecFileFull = fdata
fid.close()
if fdata[0][:2] != '#F':
self.Data['NumOfScans'] = 0
self.Data['Message'] = 'The file is not a valid specfile!!'
print 'Error:: The file is not a valid specfile!!'
else:
startScanLineNums = [
i for i in range(startLineNum, len(fdata))
if fdata[i][:2] == '#S'
]
self.progressDialog.setMaximum(len(startScanLineNums))
self.progressDialog.show()
if startLineNum > 0:
startScanLineNums = sorted(startScanLineNums)
numOfScans = len(startScanLineNums)
scanLines = [fdata[i] for i in startScanLineNums]
if startLineNum == 0:
tmp = 0
self.Data['NumOfScans'] = 0 #numOfScans
self.Data['ScanNums'] = []
offset = 0
# self.Data['ScanLines']=[]#scanLines
# self.Data['StartScanLineNums']=[]#startScanLineNums
else:
tmp = self.Data['NumOfScans']
self.Data[
'NumOfScans'] = self.Data['NumOfScans'] - 1 #+numOfScans
offset = 1
# self.Data['ScanLines']=self.Data['ScanLines'][:-1]#+scanLines[1:]
# self.Data['StartScanLineNums']=self.Data['StartScanLineNums'][:-1]#+startScanLineNums
if int(scanLines[-1].split()
[1]) != len(startScanLineNums) + endScanNum - offset:
print len(
startScanLineNums), scanLines[-1].split()[1], endScanNum
self.Data['Error'] = True
self.Data['Message'] = 'There are identical scans in the file'
else:
self.Data['Error'] = False
for i in range(numOfScans):
start = startScanLineNums[i] + 1
line = fdata[start]
num = int(fdata[startScanLineNums[i]].split()[1])
i = i + tmp
self.Data[num] = {}
self.Par[num] = {}
if fdata[start - 1].split()[2] == 'getandsave_mca':
self.Par[num]['Detector'] = 'Vortex'
self.Data[num]['ScanVar'] = 'Empty'
else:
self.Par[num]['Detector'] = 'Monitor'
tmpdata = []
while line[:2] != '\n' and line[:2] != '#C':
if line[:2] == '#P':
parName = line[4:].split()
start = start + 1
parValue = map(eval, fdata[start][1:].split())
for j in range(len(parName)):
self.Par[num][parName[j]] = parValue[j]
if line[:2] == '#W':
tmppar = line[2:].split()
self.Par[num]['Wavelength'] = eval(tmppar[1])
if line[:3] == '#G0':
self.Par[num]['g_l1'] = float(line[4:].split()[5])
self.Par[num]['g_l2'] = float(line[4:].split()[6])
self.Par[num]['g_l3'] = float(line[4:].split()[7])
if line[:2] == '#A':
tmppar = line[2:].split()
self.Par[num]['Absorber'] = eval(tmppar[1])
if line[:2] == '#Q':
tmppar = line[2:].split()
self.Par[num]['Q'] = map(eval, tmppar)
if line[:2] == '#V':
self.Par[num]['Detector'] = 'Vortex'
if line[:3] == '#B0':
tmppar = line[3:].split('.')
self.Par[num]['ImageNumber'] = len(
line[3:].split('_')[-1].split('.')[0])
if tmppar[1] == 'tif\n':
self.Par[num]['Detector'] = 'Pilatus'
elif tmppar[1] == 'sfrm\n':
self.Par[num]['Detector'] = 'Bruker'
if line[:3] == '#B1':
try:
tmppar = map(eval, line[3:].split())
except:
tmppar = map(eval, line[3:].split()[:-1])
self.Par[num]['DBPos'] = tmppar[:2]
self.Par[num]['S2D_Dist'] = tmppar[2]
self.Par[num]['S7D_Dist'] = tmppar[3]
if line[:3] == '#B2':
tmppar = map(eval, line[3:].split())
self.Par[num]['DBPos'] = tmppar[:2]
self.Par[num]['S2D_Dist'] = tmppar[2]
self.Par[num]['S7D_Dist'] = tmppar[3]
if line[:2] == '#L':
scanVar = line[3:-1].split()
self.Data[num]['ScanVar'] = scanVar
if line[0] != '#':
try:
tmpdata.append(map(eval, line.split()))
except:
self.Data[num][
'Message'] = 'Something wrong with Scan Number %d', num, '.Please check the the scan in the specfile.'
print 'Something wrong with Scan Number %d', num
start = start + 1
try:
line = fdata[start]
except:
break
if self.Data[num]['ScanVar'] != 'Empty':
for j in range(len(scanVar)):
try:
self.Data[num][scanVar[j]] = np.array(
tmpdata, dtype='float')[:, j]
except:
self.Data[num][scanVar[j]] = None
if len(self.Par[num]) == 1:
self.Par[num]['Message'] = 'No parameters!!'
self.progressDialog.setLabelText('Reading Scan #' + str(num))
self.updateProgress()
self.Data['NumOfScans'] = num
# self.Data['ScanLines']=self.Data['ScanLines']+[scanLines[num-tmp]]
self.Data[num]['ScanLine'] = fdata[startScanLineNums[i - tmp]]
self.Data[num]['StartScanLineNum'] = startScanLineNums[i - tmp]
self.endLineNum = startScanLineNums[i - tmp]
self.Data['ScanNums'].append(num)
if self.progressDialog.wasCanceled() == True:
break
self.progressDialog.hide()
def APS_9IDC(self, startLineNum=0, endScanNum=0, det='CCD'):
"""
Function to read a complete spec File collected at APS 9IDC
"""
self.progressDialog = QProgressDialog('Reading scans form SPEC File:',
'Abort', 0, 100)
self.progressDialog.setWindowModality(Qt.WindowModal)
self.progressDialog.setWindowTitle('Wait')
self.progressDialog.setAutoClose(True)
self.progressDialog.setAutoReset(True)
self.progressDialog.setMinimum(1)
self.Data['YCol'] = 'Bicron1'
self.Data['NCol'] = 'i2'
fid = open(self.specfile)
fdata = fid.readlines()
self.SpecFileFull = fdata
fid.close()
if fdata[0][:2] != '#F':
self.Data['NumOfScans'] = 0
self.Data['Message'] = 'The file is not a valid specfile!!'
print 'Error:: The file is not a valid specfile!!'
else:
startScanLineNums = [
i for i in range(startLineNum, len(fdata))
if fdata[i][:2] == '#S'
]
self.progressDialog.setMaximum(len(startScanLineNums))
self.progressDialog.show()
self.endLineNum = startScanLineNums[-1]
if startLineNum > 0:
startScanLineNums = sorted(startScanLineNums)
self.Data['StartScanLineNums'] = startScanLineNums
numOfScans = len(self.Data['StartScanLineNums'])
scanLines = [fdata[i] for i in startScanLineNums]
if startLineNum == 0:
tmp = 0
self.Data['NumOfScans'] = 0 #numOfScans
self.Data['ScanLines'] = [] #scanLines
self.Data['StartScanLineNums'] = [] #startScanLineNums
self.Par['ParName'] = []
for i in range(startScanLineNums[0]):
line = fdata[i].split()
if fdata[i][:2] == '#O':
self.Par['ParName'] = self.Par['ParName'] + line[1:]
else:
tmp = self.Data['NumOfScans']
self.Data[
'NumOfScans'] = self.Data['NumOfScans'] - 1 #+numOfScans
self.Data['ScanLines'] = self.Data[
'ScanLines'][:-1] #+scanLines[1:]
self.Data['StartScanLineNums'] = self.Data[
'StartScanLineNums'][:-1] #+startScanLineNums
for i in range(numOfScans):
start = startScanLineNums[i] + 1
line = fdata[start]
i = i + tmp
self.Data[i] = {}
self.Par[i] = {}
if fdata[start - 1].split()[2] == 'getandsave_mca' or fdata[
start - 1].split()[2] == 'MCAscanpt':
self.Par[i]['Mca'] = 1
self.Data[i]['ScanVar'] = 'Empty'
else:
self.Par[i]['Mca'] = 0
self.Par[i]['CCD'] = 0
tmpdata = []
pstart = 0
while line[:2] != '\n' and line[:2] != '#C':
if line[:2] == '#P':
parValue = map(eval, fdata[start].split()[1:])
for j in range(len(parValue)):
self.Par[i][self.Par['ParName']
[pstart]] = parValue[j]
pstart = pstart + 1
if line[:2] == '#Q':
tmppar = line[2:].split()
self.Par[i]['Q'] = map(eval, tmppar)
if line[:2] == '#L':
scanVar = line[3:-1].split()
self.Data[i]['ScanVar'] = scanVar
if line[0] != '#':
tmpdata.append(map(eval, line.split()))
start = start + 1
line = fdata[start]
for j in range(len(scanVar)):
try:
self.Data[i][scanVar[j]] = np.array(tmpdata)[:, j]
except:
self.Data[i][scanVar[j]] = None
if len(self.Par[i]) == 1:
self.Par[i]['Message'] = 'No parameters!!'
self.progressDialog.setLabelText(
'Reading scans form SPEC File: ' + str(i + 1))
self.updateProgress()
self.Data['NumOfScans'] = i
self.Data['ScanLines'] = self.Data['ScanLines'] + [
scanLines[i - tmp]
]
self.Data['StartScanLineNums'] = self.Data[
'StartScanLineNums'] + [startScanLineNums[i - tmp]]
self.endLineNum = startScanLineNums[i - tmp]
if self.progressDialog.wasCanceled() == True:
break
self.progressDialog.hide()
class PetaJakartaDialog(QDialog, FORM_CLASS):
"""Downloader for petajakarta data.
.. versionadded: 3.3
"""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
.. versionadded: 3.3
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('PetaJakarta Downloader'))
self.iface = iface
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('PetaJakarta Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up things for context help
self.help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
# Allow toggling the help button
self.help_button.setCheckable(True)
self.help_button.toggled.connect(self.help_toggled)
self.main_stacked_widget.setCurrentIndex(1)
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
self.time_stamp = None
self.restore_state()
@pyqtSlot()
@pyqtSignature('bool') # prevents actions being handled twice
def help_toggled(self, flag):
"""Show or hide the help tab in the stacked widget.
.. versionadded: 3.3
:param flag: Flag indicating whether help should be shown or hidden.
:type flag: bool
"""
if flag:
self.help_button.setText(self.tr('Hide Help'))
self.show_help()
else:
self.help_button.setText(self.tr('Show Help'))
self.hide_help()
def hide_help(self):
"""Hide the usage info from the user.
.. versionadded:: 3.3
"""
self.main_stacked_widget.setCurrentIndex(1)
def show_help(self):
"""Show usage info to the user.
.. versionadded: 3.3
"""
# Read the header and footer html snippets
self.main_stacked_widget.setCurrentIndex(0)
header = html_header()
footer = html_footer()
string = header
message = peta_jakarta_help()
string += message.to_html()
string += footer
self.help_web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory.
.. versionadded: 3.3
"""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def accept(self):
"""Do PetaJakarta download and display it in QGIS.
.. versionadded: 3.3
"""
self.save_state()
try:
self.require_directory()
except CanceledImportDialogError:
return
QtGui.qApp.setOverrideCursor(QtGui.QCursor(QtCore.Qt.WaitCursor))
source = (
'https://rem.petajakarta.org/banjir/data/api/v2/rem/flooded')
layer = QgsVectorLayer(source, 'flood', 'ogr', False)
self.time_stamp = time.strftime('%d-%b-%Y %H:%M:%S')
# Now save as shp
name = 'jakarta_flood.shp'
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_flag.isChecked()
date_stamp_flag = self.include_date_flag.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory,
output_prefix,
date_stamp_flag,
name,
overwrite)
QgsVectorFileWriter.writeAsVectorFormat(
layer, output_base_file_path, 'CP1250', None, 'ESRI Shapefile')
# Get rid of the GeoJSON layer and rather use local shp
del layer
self.copy_style(output_base_file_path)
self.copy_keywords(output_base_file_path)
layer = self.add_flooded_field(output_base_file_path)
# add the layer to the map
registry = QgsMapLayerRegistry.instance()
registry.addMapLayer(layer)
self.disable_busy_cursor()
self.done(QDialog.Accepted)
def add_flooded_field(self, shapefile_path):
"""Create the layer from the local shp adding the flooded field.
.. versionadded:: 3.3
Use this method to add a calculated field to a shapefile. The shapefile
should have a field called 'count' containing the number of flood
reports for the field. The field values will be set to 0 if the count
field is < 1, otherwise it will be set to 1.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
:return: A vector layer with the flooded field added.
:rtype: QgsVectorLayer
"""
layer = QgsVectorLayer(
shapefile_path, self.tr('Jakarta Floods'), 'ogr')
# Add a calculated field indicating if a poly is flooded or not
# from PyQt4.QtCore import QVariant
layer.startEditing()
field = QgsField('flooded', QVariant.Int)
layer.dataProvider().addAttributes([field])
layer.commitChanges()
layer.startEditing()
idx = layer.fieldNameIndex('flooded')
expression = QgsExpression('state > 0')
expression.prepare(layer.pendingFields())
for feature in layer.getFeatures():
feature[idx] = expression.evaluate(feature)
layer.updateFeature(feature)
layer.commitChanges()
return layer
def copy_keywords(self, shapefile_path):
"""Copy keywords from the OSM resource directory to the output path.
.. versionadded: 3.3
In addition to copying the template, tokens within the template will
be replaced with new values for the date token and title token.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
"""
source_xml_path = resources_path('petajakarta', 'flood-keywords.xml')
output_xml_path = shapefile_path.replace('shp', 'xml')
LOGGER.info('Copying xml to: %s' % output_xml_path)
title_token = '[TITLE]'
new_title = self.tr('Jakarta Floods - %s' % self.time_stamp)
date_token = '[DATE]'
new_date = self.time_stamp
with open(source_xml_path) as source_file, \
open(output_xml_path, 'w') as output_file:
for line in source_file:
line = line.replace(date_token, new_date)
line = line.replace(title_token, new_title)
output_file.write(line)
@staticmethod
def copy_style(shapefile_path):
"""Copy style from the OSM resource directory to the output path.
.. versionadded: 3.3
:param shapefile_path: Path to the shapefile that should get the path
added.
:type shapefile_path: basestring
"""
source_qml_path = resources_path('petajakarta', 'flood-style.qml')
output_qml_path = shapefile_path.replace('shp', 'qml')
LOGGER.info('Copying qml to: %s' % output_qml_path)
copy(source_qml_path, output_qml_path)
@staticmethod
def disable_busy_cursor():
"""Disable the hourglass cursor.
TODO: this is duplicated from dock.py
"""
while QtGui.qApp.overrideCursor() is not None and \
QtGui.qApp.overrideCursor().shape() == QtCore.Qt.WaitCursor:
QtGui.qApp.restoreOverrideCursor()
def get_output_base_path(
self,
output_directory,
output_prefix,
with_date_stamp,
feature_type,
overwrite):
"""Get a full base name path to save the shapefile.
TODO: This is cut & paste from OSM - refactor to have one method
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param with_date_stamp: Whether to add a datestamp in between the
file prefix and the feature_type for the shapefile name.
:type output_prefix: str
:param feature_type: What kind of data will be downloaded. Will be
used for the shapefile name.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
if with_date_stamp and self.time_stamp is not None:
time_stamp = self.time_stamp.replace(' ', '-')
time_stamp = time_stamp.replace(':', '-')
time_stamp += '-'
feature_type = time_stamp + feature_type
path = os.path.join(
output_directory, '%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix(
'%s.shp' % path, separator)
if suffix:
path = os.path.join(output_directory, '%s%s%s%s' % (
output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
TODO: This is cut & paste from OSM - refactor to have one method
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return PetaJakartaDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
TODO: This is cut & paste from OSM - refactor to have one method
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
TODO: This is cut & paste from OSM - refactor to have one method
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.iface,
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers')
)
def reject(self):
"""Redefinition of the reject() method.
It will call the super method.
"""
# add our own logic here...
super(PetaJakartaDialog, self).reject()
class ImportDialog(QDialog, Ui_ImportDialogBase):
def __init__(self, theParent=None, theIface=None):
"""Constructor for import dialog.
Args:
* theParent - Optional widget to use as parent
* theIface - an instance of QGisInterface
Returns:
not applicable
Raises:
no exceptions explicitly raised
"""
QDialog.__init__(self, theParent)
self.parent = theParent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = theIface
self.url = "http://osm.linfiniti.com/buildings-shp"
## region coordinate: (latitude, longtitude, zoom_level)
self.regionExtent = {
'0': [18.87685, -71.493, 6], # haiti
'1': [-2.5436300, 116.8887, 3], # indonesia
'2': [1.22449, 15.40999, 2], # africa
'3': [34.05, 56.55, 3], # middle east
'4': [12.98855, 121.7166, 4], # philipine
}
# creating progress dialog for download
self.progressDialog = QProgressDialog(self)
self.progressDialog.setAutoClose(False)
myTitle = self.tr("InaSAFE OpenStreetMap Downloader")
self.progressDialog.setWindowTitle(myTitle)
## set map parameter based on placeholder self.map widget
theMap = InasafeLightMaps(self.gbxMap)
theMap.setGeometry(self.map.geometry())
theMap.setSizePolicy(self.map.sizePolicy())
self.map = theMap
self.nam = QNetworkAccessManager(self)
self.setupOptions()
self.restoreState()
self.cbxRegion.currentIndexChanged.connect(self.regionChanged)
self.map.m_normalMap.updated.connect(self.updateExtent)
def regionChanged(self, theIndex):
""" Slot that called when region combo box changed.
Params:
theIndex - index of combo box
"""
myRegionIndex = str(self.cbxRegion.itemData(theIndex).toString())
myCenter = self.regionExtent[myRegionIndex]
self.map.setCenter(myCenter[0], myCenter[1], myCenter[2])
# pylint: disable=W0613
def resizeEvent(self, theEvent):
""" Function that called when resize event occurred.
Params:
theEvent - QEvent instance. Not used.
"""
self.map.resize(self.gbxMap.width() - 30, self.gbxMap.height() - 30)
# pylint: disable=W0613
def setupOptions(self):
""" Set the content of combo box for region and preset """
self.cbxRegion.insertItem(0, 'Indonesia', 1)
self.cbxRegion.insertItem(1, 'Africa', 2)
self.cbxRegion.insertItem(2, 'Philippines', 4)
self.cbxRegion.insertItem(3, 'Central Asia/Middle East', 3)
self.cbxRegion.insertItem(4, 'Haiti', 0)
self.cbxPreset.insertItem(0, self.tr('Buildings'), 'building')
self.cbxPreset.insertItem(0, self.tr('Highway'), 'highway')
def restoreState(self):
""" Read last state of GUI from configuration file """
mySetting = QSettings()
myRegion = mySetting.value('region').toInt()
if myRegion[1] is True:
self.cbxRegion.setCurrentIndex(myRegion[0])
myPreset = mySetting.value('preset').toInt()
if myPreset[1] is True:
self.cbxPreset.setCurrentIndex(myPreset[0])
self.outDir.setText(mySetting.value('directory').toString())
myZoomLevel = mySetting.value('zoom_level').toInt()
myLatitude = mySetting.value('latitude').toDouble()
myLongitude = mySetting.value('longitude').toDouble()
if myZoomLevel[1] is True:
self.map.setCenter(myLatitude[0], myLongitude[0], myZoomLevel[0])
else:
# just set to indonesia extent
myCenter = self.regionExtent['1']
self.map.setCenter(myCenter[0], myCenter[1], myCenter[2])
def saveState(self):
""" Store current state of GUI to configuration file """
mySetting = QSettings()
mySetting.setValue('region', self.cbxRegion.currentIndex())
mySetting.setValue('preset', self.cbxPreset.currentIndex())
mySetting.setValue('directory', self.outDir.text())
mySetting.setValue('zoom_level', self.map.getZoomLevel())
myCenter = self.map.getCenter()
mySetting.setValue('latitude', myCenter[0])
mySetting.setValue('longitude', myCenter[1])
def updateExtent(self):
""" Update extent value in GUI based from value in map widget"""
myExtent = self.map.getExtent()
self.minLongitude.setText(str(myExtent[1]))
self.minLatitude.setText(str(myExtent[0]))
self.maxLongitude.setText(str(myExtent[3]))
self.maxLatitude.setText(str(myExtent[2]))
@pyqtSignature('') # prevents actions being handled twice
def on_pBtnDir_clicked(self):
""" Show a dialog to choose directory """
self.outDir.setText(
QFileDialog.getExistingDirectory(self,
self.tr("Select Directory")))
def accept(self):
""" Do import process """
try:
self.saveState()
self.ensureDirExist()
self.doImport()
self.loadShapeFile()
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
QMessageBox.warning(
self, self.tr("InaSAFE OpenStreetMap Downloader Error"),
str(myEx))
self.progressDialog.cancel()
def ensureDirExist(self):
"""
Ensure directory path entered in dialog exist.
When the path is not exist, this function will
ask the user if he want to create it or not.
Raises:
CanceledImportDialogError - when user choose 'No'
in question dialog for creating directory.
"""
myDir = str(self.outDir.text())
if os.path.exists(myDir):
return
myTitle = self.tr("Directory %1 not exist").arg(myDir)
myQuestion = self.tr(
"Directory %1 not exist. Are you want to create it?").arg(myDir)
myAnswer = QMessageBox.question(self, myTitle, myQuestion,
QMessageBox.Yes | QMessageBox.No)
if myAnswer == QMessageBox.Yes:
os.makedirs(myDir)
else:
raise CanceledImportDialogError()
def doImport(self):
"""
Import shape files from Linfinti.
Raises:
* ImportDialogError - when network error occurred
* CanceledImportDialogError - when user press cancel button
"""
## preparing necessary data
myMinLng = str(self.minLongitude.text())
myMinLat = str(self.minLatitude.text())
myMaxLng = str(self.maxLongitude.text())
myMaxLat = str(self.maxLatitude.text())
myCurrentIndex = self.cbxPreset.currentIndex()
myType = str(self.cbxPreset.itemData(myCurrentIndex).toString())
myCoordinate = "{myMinLng},{myMinLat},{myMaxLng},{myMaxLat}".format(
myMinLng=myMinLng,
myMinLat=myMinLat,
myMaxLng=myMaxLng,
myMaxLat=myMaxLat)
myShapeUrl = "{url}?bbox={myCoordinate}&obj={type}".format(
url=self.url, myCoordinate=myCoordinate, type=myType)
myFilePath = tempfile.mktemp('.shp.zip')
# download and extract it
self.downloadShapeFile(myShapeUrl, myFilePath)
self.extractZip(myFilePath, str(self.outDir.text()))
self.progressDialog.done(QDialog.Accepted)
def downloadShapeFile(self, theUrl, theOutput):
"""
Download shape file from theUrl and write to theOutput.
Params:
* theUrl - URL of shape file
* theOutput - path of output file
Raises:
* ImportDialogError - when network error occurred
"""
self.progressDialog.show()
self.progressDialog.setMaximum(100)
self.progressDialog.setValue(0)
# myLabelText = "Begin downloading shapefile from " \
# + "%1 ..."
# self.progressDialog.setLabelText(self.tr(myLabelText).arg(theUrl))
myLabelText = self.tr("Begin downloading shapefile")
self.progressDialog.setLabelText(myLabelText)
myResult = httpDownload(self.nam, theUrl, theOutput,
self.progressDialog)
if myResult is not True:
_, myErrorMessage = myResult
raise ImportDialogError(myErrorMessage)
def extractZip(self, thePath, theOutDir):
"""
Extract all content of zip file from thePath to theOutDir.
Args:
* thePath - the path of zip file
* theOutDir - output directory
Raises:
IOError - when cannot open thePath or theOutDir is not exist.
"""
import zipfile
## extract all files...
myHandle = open(thePath, 'rb')
myZip = zipfile.ZipFile(myHandle)
for myName in myZip.namelist():
myOutPath = os.path.join(theOutDir, myName)
myOutFile = open(myOutPath, 'wb')
myOutFile.write(myZip.read(myName))
myOutFile.close()
myHandle.close()
def loadShapeFile(self):
"""
Load downloaded shape file to QGIS Main Window.
Raises:
ImportDialogError - when buildings.shp not exist
"""
myDir = str(self.outDir.text())
myPath = os.path.join(myDir, 'buildings.shp')
if not os.path.exists(myPath):
myMessage = self.tr(
"%s don't exist. The server don't have buildings data.")
raise ImportDialogError(myMessage)
self.iface.addVectorLayer(myPath, 'buildings', 'ogr')
class CpkPacker():
def __init__(self, parent=None):
self.parent = parent
self.process = None
def __pack_cpk(self, csv, cpk):
self.progress.setValue(0)
self.progress.setMaximum(100000)
self.progress.setLabelText("Building %s" % cpk)
process = QProcess()
process.start("tools/cpkmakec",
[csv, cpk, "-align=2048", "-mode=FILENAME"])
percent = 0
while not process.waitForFinished(100):
output = QString(process.readAll())
output = output.split("\n", QString.SkipEmptyParts)
for line in output:
line = common.qt_to_unicode(line)
match = OUTPUT_RE.search(line)
if match == None:
continue
percent = float(match.group(1)) * 1000
self.progress.setValue(percent)
percent += 1
def __cache_outdated(self, src_dir, cache_file):
if not os.path.isfile(cache_file):
return True
cache_updated = os.path.getmtime(cache_file)
for src_file in list_all_files(src_dir):
if os.path.getmtime(src_file) > cache_updated:
return True
return False
def create_archives(self):
try:
self.width = self.parent.width()
self.height = self.parent.height()
self.x = self.parent.x()
self.y = self.parent.y()
except:
self.width = 1920
self.height = 1080
self.x = 0
self.y = 0
self.progress = QProgressDialog("Reading...", QtCore.QString(), 0,
7600, self.parent)
self.progress.setWindowModality(Qt.Qt.WindowModal)
self.progress.setValue(0)
self.progress.setAutoClose(False)
self.progress.setMinimumDuration(0)
USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME",
"USRDIR")
eboot_path = os.path.join(common.editor_config.iso_dir, "PSP_GAME",
"SYSDIR", "EBOOT.BIN")
eboot = BitStream(filename=eboot_path)
eboot = eboot_patch.apply_eboot_patches(eboot)
# So we can loop. :)
ARCHIVE_INFO = [
{
"dir":
common.editor_config.data00_dir,
"cpk":
os.path.join(USRDIR, "data00.cpk"),
"csv":
os.path.join(
"data",
"data00.csv" if not common.editor_config.quick_build else
"data00-quick.csv"),
"name":
"data00.cpk",
"pack":
common.editor_config.pack_data00,
},
{
"dir":
common.editor_config.data01_dir,
"cpk":
os.path.join(USRDIR, "data01.cpk"),
"csv":
os.path.join(
"data",
"data01.csv" if not common.editor_config.quick_build else
"data01-quick.csv"),
"name":
"data01.cpk",
"pack":
common.editor_config.pack_data01,
},
]
# temp_dir = tempfile.mkdtemp(prefix = "sdse-")
temp_dir = common.editor_config.build_cache
for archive in ARCHIVE_INFO:
if not archive["pack"]:
continue
self.progress.setWindowTitle("Building " + archive["name"])
csv_template_f = open(archive["csv"], "rb")
csv_template = csv.reader(csv_template_f)
csv_out_path = os.path.join(temp_dir, "cpk.csv")
csv_out_f = open(csv_out_path, "wb")
csv_out = csv.writer(csv_out_f)
for row in csv_template:
if len(row) < 4:
continue
base_path = row[0]
real_path = os.path.join(archive["dir"], base_path)
out_path = os.path.join(temp_dir, archive["name"], base_path)
self.progress.setValue(self.progress.value() + 1)
self.progress.setLabelText("Reading...\n%s" % real_path)
# All items in the CPK list should be files.
# Therefore, if we have a directory, then it needs to be packed.
if os.path.isdir(real_path):
if self.__cache_outdated(real_path, out_path):
out_dir = os.path.dirname(out_path)
try:
os.makedirs(out_dir)
except:
pass
data = pack_dir(real_path)
with open(out_path, "wb") as out_file:
data.tofile(out_file)
del data
elif os.path.isfile(real_path):
# If it's a file, though, we can just use it directly.
out_path = real_path
row[0] = out_path
csv_out.writerow(row)
csv_template_f.close()
csv_out_f.close()
self.__pack_cpk(csv_out_path, archive["cpk"])
self.progress.setWindowTitle("Building...")
self.progress.setLabelText("Saving EBOOT.BIN...")
self.progress.setValue(self.progress.maximum())
with open(eboot_path, "wb") as f:
eboot.tofile(f)
# self.progress.setLabelText("Deleting temporary files...")
# shutil.rmtree(temp_dir)
self.progress.close()
class OsmDownloader(QDialog, Ui_OsmDownloaderBase):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.url = "http://osm.linfiniti.com/buildings-shp"
# creating progress dialog for download
self.progressDialog = QProgressDialog(self)
self.progressDialog.setAutoClose(False)
myTitle = self.tr("InaSAFE OpenStreetMap Downloader")
self.progressDialog.setWindowTitle(myTitle)
self.show_info()
self.network_manager = QNetworkAccessManager(self)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') data from the '
'OpenStreetMap project for you. The downloaded data will have '
'InaSAFE keywords defined and a default QGIS style applied. To '
'use this tool effectively:')
tips = m.BulletedList()
tips.add(
self.
tr('Use QGIS to zoom in to the area for which you want building data '
'to be retrieved.'))
tips.add(
self.
tr('Check the output directory is correct. Note that the saved '
'dataset will be called buildings.shp (and its associated files).'
))
tips.add(
self.
tr('If a dataset already exists in the output directory it will be '
'overwritten.'))
tips.add(
self.
tr('This tool requires a working internet connection and fetching '
'buildings will consume your bandwidth.'))
tips.add(
m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.webView.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
mySetting = QSettings()
self.outDir.setText(mySetting.value('directory').toString())
def save_state(self):
""" Store current state of GUI to configuration file """
mySetting = QSettings()
mySetting.setValue('directory', self.outDir.text())
def update_extent(self):
""" Update extent value in GUI based from value in map."""
myExtent = self.iface.mapCanvas().extent()
self.minLongitude.setText(str(myExtent.xMinimum()))
self.minLatitude.setText(str(myExtent.yMinimum()))
self.maxLongitude.setText(str(myExtent.xMaximum()))
self.maxLatitude.setText(str(myExtent.yMaximum()))
@pyqtSignature('') # prevents actions being handled twice
def on_pBtnDir_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.outDir.setText(
QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
try:
self.save_state()
self.require_directory()
self.download()
self.load_shapefile()
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self, self.tr("InaSAFE OpenStreetMap downloader error"),
str(myEx))
self.progressDialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
myDir = str(self.outDir.text())
if os.path.exists(myDir):
return
myTitle = self.tr("Directory %1 not exist").arg(myDir)
myQuestion = self.tr(
"Directory %1 not exist. Are you want to create it?").arg(myDir)
# noinspection PyCallByClass,PyTypeChecker
myAnswer = QMessageBox.question(self, myTitle, myQuestion,
QMessageBox.Yes | QMessageBox.No)
if myAnswer == QMessageBox.Yes:
os.makedirs(myDir)
else:
raise CanceledImportDialogError()
def download(self):
"""Download shapefiles from Linfinti server.
:raises: ImportDialogError, CanceledImportDialogError
"""
## preparing necessary data
myMinLng = str(self.minLongitude.text())
myMinLat = str(self.minLatitude.text())
myMaxLng = str(self.maxLongitude.text())
myMaxLat = str(self.maxLatitude.text())
myCoordinate = "{myMinLng},{myMinLat},{myMaxLng},{myMaxLat}".format(
myMinLng=myMinLng,
myMinLat=myMinLat,
myMaxLng=myMaxLng,
myMaxLat=myMaxLat)
myShapeUrl = "{url}?bbox={myCoordinate}".format(
url=self.url, myCoordinate=myCoordinate)
myFilePath = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(myShapeUrl, myFilePath)
print myFilePath
print str(self.outDir.text())
self.extract_zip(myFilePath, str(self.outDir.text()))
self.progressDialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
self.progressDialog.show()
self.progressDialog.setMaximum(100)
self.progressDialog.setValue(0)
# myLabelText = "Begin downloading shapefile from " \
# + "%1 ..."
# self.progressDialog.setLabelText(self.tr(myLabelText).arg(url))
myLabelText = self.tr("Downloading shapefile")
self.progressDialog.setLabelText(myLabelText)
myResult = download_url(self.network_manager, url, output_path,
self.progressDialog)
if myResult is not True:
_, myErrorMessage = myResult
raise ImportDialogError(myErrorMessage)
def extract_zip(self, path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
## extract all files...
myHandle = open(path, 'rb')
myZip = zipfile.ZipFile(myHandle)
for myName in myZip.namelist():
myOutPath = os.path.join(output_dir, myName)
myOutFile = open(myOutPath, 'wb')
myOutFile.write(myZip.read(myName))
myOutFile.close()
myHandle.close()
def load_shapefile(self):
"""
Load downloaded shape file to QGIS Main Window.
:raises: ImportDialogError - when buildings.shp not exist
"""
myDir = str(self.outDir.text())
myPath = os.path.join(myDir, 'buildings.shp')
if not os.path.exists(myPath):
myMessage = self.tr(
"%s don't exist. The server don't have buildings data.")
raise ImportDialogError(myMessage)
self.iface.addVectorLayer(myPath, 'buildings', 'ogr')
def export_umdimage(src, dst, convert_gim = True, unique = False, parent = None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError("Cannot export %s. Source and destination directories are the same." % src)
answer = QtGui.QMessageBox.question(
parent,
"Export Directory",
"Exporting directory:\n\n" + src + "\n\n" +
"into directory:\n\n" + dst + "\n\n" +
"Proceed?",
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton = QtGui.QMessageBox.No
)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent)
progress.setWindowTitle("Exporting...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
if parent:
width = parent.width()
height = parent.height()
x = parent.x()
y = parent.y()
else:
width = 1920
height = 1080
x = 0
y = 0
seen_groups = []
count = 0
last_update = time.time()
progress.setMaximum(60000)
for filename in list_all_files(src):
if progress.wasCanceled():
break
count += 1
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Exporting...\n" + filename)
progress.setValue(count)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
base_name = filename[len(src) + 1:]
if unique:
dupe_name = os.path.join("umdimage", base_name)
dupe_name = os.path.normpath(os.path.normcase(dupe_name))
group = _DUPE_DB.group_from_file(dupe_name)
if group in seen_groups:
continue
if not group == None:
seen_groups.append(group)
dst_file = os.path.join(dst, base_name)
dst_dir = os.path.dirname(dst_file)
ext = os.path.splitext(dst_file)[1].lower()
try:
os.makedirs(dst_dir)
except:
pass
if ext == ".gim" and convert_gim:
dst_file = os.path.splitext(dst_file)[0] + ".png"
_CONV.gim_to_png(filename, dst_file)
else:
shutil.copy2(filename, dst_file)
progress.close()
def calculate_progress(parent, filter = script_analytics.DEFAULT_FILTER):
start_time = time.time()
progress = QProgressDialog("Calculating translation progress...", "Abort", 0, 72000, parent)
progress.setWindowTitle("Translation Progress")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
# For our dupe database, we need "umdimage" instead of wherever the files
# are really stored, so we strip that part off first.
dir_start = len(common.editor_config.umdimage_dir) + 1
total_files = 0
unique_files = 0
translated_files = 0
translated_unique = 0
total_chars = 0
unique_chars = 0
translated_chars = 0
translated_unique_chars = 0
translated_words = 0
translated_unique_words = 0
total_bytes = 0
unique_bytes = 0
translated_bytes = 0
translated_unique_bytes = 0
groups_seen = set()
files_seen = set()
untranslated_lines = []
for i, total, filename, data in script_analytics.SA.get_data(filter):
if progress.wasCanceled():
return
progress.setValue(i)
progress.setMaximum(total)
if data == None:
continue
db_name = os.path.join("umdimage", filename)
real_name = os.path.join(common.editor_config.umdimage_dir, filename)
if db_name in files_seen:
continue
dupe_group = dupe_db.db.group_from_file(db_name)
# Add the whole group to the translated files, but only one
# to the unique translated. If there is no group, it's size 1.
group_size = 1
if not dupe_group == None:
if dupe_group in groups_seen:
continue
else:
groups_seen.add(dupe_group)
group_files = dupe_db.db.files_in_group(dupe_group)
group_size = 0
for dupe_file in group_files:
#if filter.search(dupe_file):
group_size += 1
files_seen.update(group_files)
total_files += group_size
unique_files += 1
#file = script_for_counting(data)
file = data
# How many characters is the untranslated, non-tagged text?
num_chars = len(file.original_notags)
#num_bytes = len(bytearray(file.original_notags, encoding = "SJIS", errors = "replace"))
total_chars += num_chars * group_size
unique_chars += num_chars
#total_bytes += num_bytes * group_size
#unique_bytes += num_bytes
if not file.translated_notags == "" or num_chars == 0:
translated_files += group_size
translated_unique += 1
translated_chars += num_chars * group_size
translated_unique_chars += num_chars
words = count_words(file.translated_notags)
translated_words += words * group_size
translated_unique_words += words
#translated_bytes += num_bytes * group_size
#translated_unique_bytes += num_bytes
#elif file.translated_notags == "":
#untranslated_lines.append(db_name)
progress.close()
#print "Took %s seconds." % (time.time() - start_time)
files_percent = 100.0 if total_files == 0 else float(translated_files) / total_files * 100
unique_files_percent = 100.0 if unique_files == 0 else float(translated_unique) / unique_files * 100
chars_percent = 100.0 if total_chars == 0 else float(translated_chars) / total_chars * 100
unique_chars_percent = 100.0 if unique_chars == 0 else float(translated_unique_chars) / unique_chars * 100
bytes_percent = 100.0 if total_bytes == 0 else float(translated_bytes) / total_bytes * 100
unique_bytes_percent = 100.0 if unique_bytes == 0 else float(translated_unique_bytes) / unique_bytes * 100
QtGui.QMessageBox.information(
parent,
"Translation Progress",
("Files: %d / %d (%0.2f%%)\n" % (translated_files, total_files, files_percent)) +
("Unique Files: %d / %d (%0.2f%%)\n" % (translated_unique, unique_files, unique_files_percent)) +
"\n" +
("Japanese Characters: %d / %d (%0.2f%%)\n" % (translated_chars, total_chars, chars_percent)) +
("Unique Characters: %d / %d (%0.2f%%)\n" % (translated_unique_chars, unique_chars, unique_chars_percent)) +
#"\n" +
#("Bytes: %d / %d (%0.2f%%)\n" % (translated_bytes, total_bytes, bytes_percent)) +
#("Unique Bytes: %d / %d (%0.2f%%)\n" % (translated_unique_bytes, unique_bytes, unique_bytes_percent)) +
"\n" +
("English Words: %d\n" % (translated_words)) +
("Unique Words: %d\n" % (translated_unique_words)) +
"\n" +
"NOTE: Unique X is lazy for \"X in all unique files\"\n" +
"and not what it seems to imply." +
"",
buttons = QtGui.QMessageBox.Ok,
defaultButton = QtGui.QMessageBox.Ok
)
def export_umdimage2(src, dst, convert_gim = True, unique = False, parent = None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError("Cannot export %s. Source and destination directories are the same." % src)
answer = QtGui.QMessageBox.question(
parent,
"Export Directory",
"Exporting directory:\n\n" + src + "\n\n" +
"into directory:\n\n" + dst + "\n\n" +
"Proceed?",
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton = QtGui.QMessageBox.No
)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent)
progress.setWindowTitle("Exporting...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
if unique:
tmp_dst = tempfile.mkdtemp(prefix = "sdse-")
else:
tmp_dst = dst
seen_groups = []
for pak in glob.iglob(os.path.join(src, "bg_*.pak")):
if progress.wasCanceled():
break
pak_name = os.path.basename(pak)
out_dir = os.path.join(tmp_dst, pak_name)
progress.setLabelText("Extracting:\n" + pak)
thread = threading.Thread(target = extract_model_pak, args = (pak, out_dir, convert_gim))
thread.start()
while thread.isAlive():
thread.join(MIN_INTERVAL)
progress.setValue(progress.value() ^ 1)
if progress.wasCanceled():
progress.setLabelText("Canceling...")
if progress.wasCanceled():
break
if unique:
for img in list_all_files(out_dir):
img_base = img[len(tmp_dst) + 1:]
dupe_name = os.path.splitext(img_base)[0] + ".gim"
dupe_name = os.path.join("umdimage2", dupe_name)
dupe_name = os.path.normpath(os.path.normcase(dupe_name))
group = _DUPE_DB.group_from_file(dupe_name)
if group in seen_groups:
continue
if not group == None:
seen_groups.append(group)
dst_file = os.path.join(dst, img_base)
dst_dir = os.path.dirname(dst_file)
try:
os.makedirs(dst_dir)
except:
pass
shutil.copy2(img, dst_file)
shutil.rmtree(out_dir)
if unique:
shutil.rmtree(tmp_dst)
progress.close()
class isochronesDialog(QtGui.QDialog, FORM_CLASS):
def __init__(self, parent=None, iface=None):
"""Constructor."""
QDialog.__init__(self, parent)
# Set up the user interface from Designer.
# After setupUI you can access any designer object by doing
# self.<objectname>, and you can use autoconnect slots - see
# http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html
# #widgets-and-dialogs-with-auto-connect
self.parent = parent
self.iface = iface
self.setupUi(self)
# Setting up progress window
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('Progress')
self.progress_dialog.setWindowTitle(title)
self.restore_state()
self.canvas = iface.mapCanvas()
def accept(self):
"""Create an isochrone map and display it in QGIS."""
error_dialog_title = self.tr("Error")
try:
self.save_state()
self.require_input()
database_name = self.database.text()
host_name = self.host.text()
port_number = self.port.text()
user_name = self.user_name.text()
password = self.password.text()
network_table = self.network_table.text()
network_geom = self.network_geom_column.text()
network_id_column = self.network_id_column.text()
catchment_geom = self.catchment_geom_column.text()
catchment_table = self.catchment_table.text()
catchment_id_column = self.catchment_id_column.text()
contour_interval = self.contour_interval.text()
if self.style.isChecked():
style_checked = True
else:
style_checked = False
isochrone(
database_name,
host_name,
port_number,
user_name,
password,
network_table,
network_geom,
network_id_column,
catchment_table,
catchment_geom,
catchment_id_column,
style_checked,
contour_interval,
self,
self.progress_dialog)
self.done(QDialog.Accepted)
except ImportDialogError as exception:
display_warning_message_box(
self, error_dialog_title, exception.message)
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, error_dialog_title, exception.message)
pass
finally:
dialog_title = self.tr("Success")
def require_input(self):
"""Ensure input files are entered in dialog exist.
:raises: ImportDialogError - when one or all
of the input files are empty
"""
database_name = self.database.text()
host_name = self.host.text()
port_number = self.port.text()
user_name = self.user_name.text()
password = self.password.text()
network_table = self.network_table.text()
catchment_table = self.catchment_table.text()
contour_interval = self.contour_interval.text()
if database_name and host_name and port_number and \
user_name and password and network_table and \
catchment_table:
return
display_warning_message_box(
self,
self.tr('Error'),
self.tr('Input cannot be empty.'))
raise ImportDialogError()
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
database_name = settings.value('database', type=str)
host_name = settings.value('host', type=str)
port_number = settings.value('port', type=str)
user_name = settings.value('user_name', type=str)
network_table = settings.value('network_table', type=str)
network_geom_column = settings.value(
'network_geom_column',
type=str)
network_id_column = settings.value('network_id_column', type=str)
catchment_table = settings.value('catchment_table', type=str)
catchment_geom_column = settings.value(
'catchment_geom_column',
type=str)
catchment_id_column = settings.value(
'catchment_id_column',
type=str)
contour_interval = settings.value(
'contour_interval',
type=str)
except TypeError:
database_name = ''
host_name = ''
port_number = ''
user_name = ''
network_table = ''
network_geom_column = ''
network_id_column = ''
catchment_table = ''
catchment_geom_column = ''
catchment_id_column = ''
contour_interval = ''
self.database.setText(database_name)
self.host.setText(host_name)
self.port.setText(port_number)
self.user_name.setText(user_name)
self.network_table.setText(network_table)
self.network_geom_column.setText(network_geom_column)
self.network_id_column.setText(network_id_column)
self.catchment_table.setText(catchment_table)
self.catchment_geom_column.setText(catchment_geom_column)
self.catchment_id_column.setText(catchment_id_column)
self.contour_interval.setText(contour_interval)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('database', self.database.text())
settings.setValue('host', self.host.text())
settings.setValue('port', self.port.text())
settings.setValue('user_name', self.user_name.text())
settings.setValue('network_table', self.network_table.text())
settings.setValue(
'network_geom_column',
self.network_geom_column.text())
settings.setValue(
'network_id_column',
self.network_id_column.text())
settings.setValue('catchment_table', self.catchment_table.text())
settings.setValue(
'catchment_geom_column',
self.catchment_geom_column.text())
settings.setValue(
'catchment_id_column',
self.catchment_id_column.text())
settings.setValue(
'contour_interval',
self.contour_interval.text())
def reject(self):
"""Redefinition of the reject() method
"""
super(isochronesDialog, self).reject()
def load_isochrone_map(self, base_path):
"""Load the isochrone map in the qgis
:param base_path: Output path where layers are
:type base_path:str
"""
if not os.path.exists(base_path):
message = self.tr("Error, failed to load the isochrone map")
raise FileMissingError(message)
else:
for layer in os.listdir(base_path):
layer_name = QFileInfo(layer).baseName
if file.endswith(".asc"):
self.iface.addRasterLayer(file, layer_name)
continue
elif file.endswith(".shp"):
self.iface.addVectorLayer(file, layer_name, 'ogr')
continue
else:
continue
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_box(
self.iface,
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326.'
'You should enable \'on the fly\' to display '
'correctly the isochrone map')
)
class DatPacker():
def __init__(self, parent=None):
self.file_count = 0
self.parent = parent
def create_archives(self):
try:
self.width = self.parent.width()
self.height = self.parent.height()
self.x = self.parent.x()
self.y = self.parent.y()
except:
self.width = 1920
self.height = 1080
self.x = 0
self.y = 0
self.file_count = 0
self.progress = QProgressDialog("Reading...", QtCore.QString(), 0,
72000, self.parent)
self.progress.setWindowModality(Qt.Qt.WindowModal)
self.progress.setValue(0)
self.progress.setAutoClose(False)
self.progress.setMinimumDuration(0)
# with open(common.editor_config.eboot_orig, "rb") as f:
with open(
os.path.join(common.editor_config.iso_dir, "PSP_GAME",
"SYSDIR", "EBOOT.BIN"), "rb") as f:
eboot = BitStream(bytes=f.read())
eboot, eboot_offset = eboot_patch.apply_eboot_patches(eboot)
USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME",
"USRDIR")
# So we can loop. :)
ARCHIVE_INFO = [
{
"toc": UMDIMAGES.umdimage,
"dir": common.editor_config.umdimage_dir,
"dat": os.path.join(USRDIR, "umdimage.dat"),
"name": "umdimage.dat",
"pack": common.editor_config.pack_umdimage,
"eof": False,
},
{
"toc": UMDIMAGES.umdimage2,
"dir": common.editor_config.umdimage2_dir,
"dat": os.path.join(USRDIR, "umdimage2.dat"),
"name": "umdimage2.dat",
"pack": common.editor_config.pack_umdimage2,
"eof": False,
},
{
"toc": None,
"dir": common.editor_config.voice_dir,
"dat": os.path.join(USRDIR, "voice.pak"),
"name": "voice.pak",
"pack": common.editor_config.pack_voice,
"eof": True,
},
{
"toc": None,
"dir": common.editor_config.bgm_dir,
"dat": os.path.join(USRDIR, "bgm.pak"),
"name": "bgm.pak",
"pack": common.editor_config.pack_bgm,
"eof": True,
},
]
for archive in ARCHIVE_INFO:
if not archive["pack"]:
continue
self.progress.setWindowTitle("Building " + archive["name"])
toc_info = {}
file_list = None
if archive["toc"]:
file_list = []
toc = get_toc(eboot, archive["toc"])
for entry in toc:
filename = entry["filename"]
pos_pos = entry["file_pos_pos"]
len_pos = entry["file_len_pos"]
toc_info[filename] = [pos_pos, len_pos]
file_list.append(filename)
# Causes memory issues if I use the original order, for whatever reason.
file_list = None
with io.FileIO(archive["dat"], "w") as handler:
table_of_contents = self.pack_dir(archive["dir"],
handler,
file_list=file_list,
eof=archive["eof"])
# We're playing fast and loose with the file count anyway, so why not?
self.file_count += 1
self.progress.setValue(self.file_count)
self.progress.setLabelText("Saving " + archive["name"] + "...")
if archive["toc"]:
for entry in table_of_contents:
if not entry in toc_info:
_LOGGER.warning(
"%s missing from %s table of contents." %
(entry, archive["name"]))
continue
file_pos = table_of_contents[entry]["pos"]
file_size = table_of_contents[entry]["size"]
eboot.overwrite(BitStream(uintle=file_pos, length=32),
toc_info[entry][0] * 8)
eboot.overwrite(BitStream(uintle=file_size, length=32),
toc_info[entry][1] * 8)
del table_of_contents
self.progress.setLabelText("Saving EBOOT.BIN...")
self.progress.setValue(self.progress.maximum())
# Text replacement
to_replace = eboot_text.get_eboot_text()
for replacement in to_replace:
orig = bytearray(replacement.orig, encoding=replacement.enc)
# If they left something blank, write the original text back.
if len(replacement.text) == 0:
data = orig
else:
data = bytearray(replacement.text, encoding=replacement.enc)
pos = replacement.pos.int + eboot_offset
padding = len(orig) - len(data)
if padding > 0:
# Null bytes to fill the rest of the space the original took.
data.extend(bytearray(padding))
data = ConstBitStream(bytes=data)
eboot.overwrite(data, pos * 8)
eboot_out = os.path.join(common.editor_config.iso_dir, "PSP_GAME",
"SYSDIR", "EBOOT.BIN")
with open(eboot_out, "wb") as f:
eboot.tofile(f)
self.progress.close()
def pack_dir(self,
dir,
handler,
file_list=None,
align_toc=16,
align_files=16,
eof=False):
table_of_contents = {}
if file_list == None:
file_list = sorted(os.listdir(dir))
num_files = len(file_list)
toc_length = (num_files + 1) * 4
if eof:
toc_length += 1
if toc_length % align_toc > 0:
toc_length += align_toc - (toc_length % align_toc)
handler.seek(0)
handler.write(struct.pack("<I", num_files))
handler.write(bytearray(toc_length - 4))
for file_num, item in enumerate(file_list):
full_path = os.path.join(dir, item)
if os.path.isfile(full_path):
basename = os.path.basename(item)
basename, ext = os.path.splitext(basename)
# Special handling for certain data types.
if ext == ".txt":
data = self.pack_txt(full_path)
# anagram_81.dat is not a valid anagram file. <_>
elif basename[:
8] == "anagram_" and ext == ".dat" and not basename == "anagram_81":
anagram = AnagramFile(full_path)
data = anagram.pack(for_game=True).bytes
else:
with open(full_path, "rb") as f:
data = f.read()
else:
temp_align_toc = 16
temp_align_files = 4
if item in SPECIAL_ALIGN:
temp_align_toc = SPECIAL_ALIGN[item][0]
temp_align_files = SPECIAL_ALIGN[item][1]
elif os.path.basename(dir) in SPECIAL_ALIGN and len(
SPECIAL_ALIGN[os.path.basename(dir)]) == 4:
temp_align_toc = SPECIAL_ALIGN[os.path.basename(dir)][2]
temp_align_files = SPECIAL_ALIGN[os.path.basename(dir)][3]
if os.path.splitext(full_path)[1].lower() == ".lin":
data = self.pack_lin(full_path)
else:
data = io.BytesIO()
with io.BufferedWriter(data) as fh:
self.pack_dir(full_path,
fh,
align_toc=temp_align_toc,
align_files=temp_align_files,
eof=eof)
fh.flush()
data = data.getvalue()
data = bytearray(data)
file_size = len(data)
padding = 0
if file_size % align_files > 0:
padding = align_files - (file_size % align_files)
data.extend(bytearray(padding))
handler.seek(0, io.SEEK_END)
file_pos = handler.tell()
handler.write(data)
handler.seek((file_num + 1) * 4)
handler.write(struct.pack("<I", file_pos))
del data
self.file_count += 1
if self.file_count % 25 == 0:
self.progress.setLabelText("Reading...\n" + full_path)
self.progress.setValue(self.file_count)
# Re-center the dialog.
progress_w = self.progress.geometry().width()
progress_h = self.progress.geometry().height()
new_x = self.x + ((self.width - progress_w) / 2)
new_y = self.y + ((self.height - progress_h) / 2)
self.progress.move(new_x, new_y)
table_of_contents[item] = {}
table_of_contents[item]["size"] = file_size
table_of_contents[item]["pos"] = file_pos
if eof:
handler.seek(0, io.SEEK_END)
archive_len = handler.tell()
handler.seek((num_files + 1) * 4)
handler.write(struct.pack("<I", archive_len))
return table_of_contents
def pack_txt(self, filename):
if os.path.basename(os.path.dirname(filename)) in SCRIPT_NONSTOP:
is_nonstop = True
else:
is_nonstop = False
text = text_files.load_text(filename)
text = RE_SCRIPT.sub(u"\g<1>", text)
# Nonstop Debate lines need an extra newline at the end
# so they show up in the backlog properly.
if is_nonstop and not text[-1] == "\n":
text += "\n"
return SCRIPT_BOM.bytes + bytearray(
text, encoding="UTF-16LE") + SCRIPT_NULL.bytes
def pack_lin(self, dir):
# Collect our files.
file_list = sorted(list_all_files(dir))
txt = [
filename for filename in file_list
if os.path.splitext(filename)[1].lower() == ".txt"
]
wrd = [
filename for filename in file_list
if os.path.splitext(filename)[1].lower() == ".wrd"
]
py = [
filename for filename in file_list
if os.path.splitext(filename)[1].lower() == ".py"
]
# If there are more than one for whatever reason, just take the first.
# We only have use for a single wrd or python file.
wrd = wrd[0] if wrd else None
py = py[0] if py else None
# Prepare our temporary output directory.
temp_dir = tempfile.mkdtemp(prefix="sdse-")
# Where we're outputting our wrd file, regardless of whether it's a python
# file or a raw binary data file.
wrd_dst = os.path.join(temp_dir, "0.scp.wrd")
if py:
# _LOGGER.info("Compiling %s to binary." % py)
try:
wrd_file = WrdFile(py)
except:
_LOGGER.warning(
"%s failed to compile. Parsing wrd file instead. Exception info:\n%s"
% (py, traceback.format_exc()))
shutil.copy(wrd, wrd_dst)
else:
# If we succeeded in loading the python file, compile it to binary.
# wrd_file.save_bin(wrd)
wrd_file.save_bin(wrd_dst)
else:
shutil.copy(wrd, wrd_dst)
# Pack the text files in-place to save us a bunch of copying
# and then move it to the tmp directory with the wrd file.
if txt:
with io.FileIO(os.path.join(temp_dir, "1.dat"), "w") as h:
self.pack_dir(dir, h, file_list=txt)
# Then pack it like normal.
data = io.BytesIO()
with io.BufferedWriter(data) as h:
self.pack_dir(temp_dir, h)
h.flush()
data = data.getvalue()
shutil.rmtree(temp_dir)
return data
class MainWindow(QtGui.QWidget):
label_text_update_require = pyqtSignal(str)
def __init__(self):
super(MainWindow, self).__init__()
self.params = ParamsWidget()
self.btn_run = QtGui.QPushButton(u"Запустити")
self.progress_dialog = QProgressDialog(u'', u'Зупинити',
0, 100, self)
self.progress_dialog.setWindowModality(Qt.WindowModal)
self.progress_dialog.setAutoClose(False)
self.progress_dialog_base_title = None
self.setWindowTitle(u'Аналіз моделі')
self.init_layout()
self.init_events()
self.adjustSize()
self.move(QApplication.desktop().screen().rect().center()
- self.rect().center())
self.update_widgets_state()
def init_layout(self):
vbox_layout = QtGui.QVBoxLayout()
vbox_layout.addWidget(self.params)
vbox_layout.addStretch(1)
vbox_layout.addWidget(self.btn_run)
self.setLayout(vbox_layout)
def init_events(self):
self.params.updated.connect(self.update_widgets_state)
self.btn_run.clicked.connect(self.calculate)
self.label_text_update_require.connect(self.update_progressdialog_label)
def update_widgets_state(self):
params_ready = self.params.is_params_specified()
self.btn_run.setEnabled(params_ready)
def calculate(self):
self.run_job(u'Обробка файла...', self.process)
def run_job(self, title, target, args=None, result_handler=None):
self.progress_dialog.setWindowTitle(title)
self.progress_dialog.setValue(0)
self.calc = Calculator(target=target, args=args,
clean_up_handler=self.calc_finished,
result_handler=result_handler)
self.calc.updated.connect(self.progress_dialog.setValue)
self.calc.eta_updated.connect(self.update_eta_label)
self.calc.finished.connect(self.calc_finished)
self.progress_dialog.canceled.connect(self.calc.cancel)
self.calc.start()
@pyqtSlot()
def calc_finished(self):
self.progress_dialog.setAutoClose(True)
self.progress_dialog.setValue(100)
def process(self, update_percentage=None, check_cancelled=None):
params = self.params.get_params()
model_path = str(params['model_path'])
filename, extension = os.path.splitext(model_path)
if extension == '.obj':
self.prepare_progressdialog(u'Імпорт файла... ')
faces = importutils.get_faces(model_path, update_percentage,
check_cancelled)
print 'Faces imported'
self.prepare_progressdialog(u'Перетворення трикутників... ')
triangles = importutils.build_triangles(faces, update_percentage,
check_cancelled)
self.prepare_progressdialog(u'Корекція моделі... ')
clean_triangles = importutils.discard_invalid_triangles(triangles,
vector.Vector(20, 2, 20),
update_percentage,
check_cancelled)
print 'Triangles: %d, clean triangles: %d, diff: %d' % (
len(triangles), len(clean_triangles), len(triangles) - len(clean_triangles))
pickle_path = model_path + '.pickle'
with open(pickle_path, 'wb') as pickle_file:
try:
pickle.dump(clean_triangles, pickle_file, -1)
print 'Pickle saved to %s' % pickle_path
except Exception as e:
print 'Failed to save clean triangles list: %s' % e
elif extension == '.pickle':
with open(model_path) as pickle_file:
clean_triangles = pickle.load(pickle_file)
else:
raise Exception('Invalid extension %s' % extension)
triangles = clean_triangles
self.prepare_progressdialog(u'Обчислення E... ')
E, sum_cos, sum_sin = processor.calculate_viewpoint_sums(triangles, params['wavelength'],
vector.Vector(20, 2, 20),
update_percentage, check_cancelled)
def prepare_progressdialog(self, label):
self.progress_dialog_base_title = label
self.label_text_update_require.emit('')
self.calc.mark_process_time()
@pyqtSlot(str)
def update_progressdialog_label(self, label):
if not label:
label = self.progress_dialog_base_title
self.progress_dialog.setLabelText(label)
@pyqtSlot(float)
def update_eta_label(self, seconds):
if seconds == 0:
self.label_text_update_require.emit('')
return
t = time.gmtime(seconds)
time_string = time.strftime('%M:%S', t)
label_text = '%s %s' % (self.progress_dialog_base_title,
time_string)
self.label_text_update_require.emit(label_text)
def import_umdimage(src, dst, convert_png = True, propogate = True, parent = None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError("Cannot import %s. Source and destination directories are the same." % src)
answer = QtGui.QMessageBox.question(
parent,
"Import Directory",
"Importing directory:\n\n" + src + "\n\n" +
"into directory:\n\n" + dst + "\n\n" +
"Any affected files will be backed up. Proceed?",
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton = QtGui.QMessageBox.No
)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Finding files...", "Cancel", 0, 1, parent)
progress.setWindowTitle("Importing...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
if parent:
width = parent.width()
height = parent.height()
x = parent.x()
y = parent.y()
else:
width = 1920
height = 1080
x = 0
y = 0
progress.setMaximum(0)
progress.setValue(0)
# The raw list of files we're importing.
files = []
# A list of lists, including all dupes of the files being imported, too.
affected_files = []
file_count = 0
dupe_base = "umdimage"
tmp = tempfile.mkdtemp(prefix = "sdse-")
seen_groups = []
count = 0
last_update = time.time()
for file in list_all_files(src):
if progress.wasCanceled():
break
# Strip our base directory, so we have just a relative file list.
file = os.path.normpath(os.path.normcase(file[len(src) + 1:]))
files.append(file)
count += 1
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Finding files...\n" + file)
# progress.setValue(count)
progress.setValue(progress.value() ^ 1)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
affected_files.append([])
if os.path.splitext(file)[1] == ".png" and convert_png and file not in SKIP_CONV:
file = os.path.splitext(file)[0] + ".gim"
if propogate:
file_group = _DUPE_DB.group_from_file(os.path.join(dupe_base, file))
else:
file_group = None
if file_group in seen_groups:
continue
# If there are no dupes, just add this file.
if file_group == None:
affected_files[-1].append(file)
file_count += 1
continue
seen_groups.append(file_group)
for dupe in _DUPE_DB.files_in_group(file_group):
# Minus the "umdimage" part
dupe = dupe[len(dupe_base) + 1:]
affected_files[-1].append(dupe)
file_count += 1
progress.setValue(0)
progress.setMaximum(file_count)
# Make a backup first.
backup_dir = None
count = 0
for file_set in affected_files:
if progress.wasCanceled():
break
for file in file_set:
if progress.wasCanceled():
break
count += 1
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Backing up...\n" + file)
progress.setValue(count)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
# It's perfectly possible we want to import some files that
# don't already exist. Such as when importing a directory
# with added lines.
if not os.path.isfile(os.path.join(dst, file)):
continue
backup_dir = backup_files(dst, [file], suffix = "_IMPORT", backup_dir = backup_dir)
progress.setValue(0)
# And now do our importing.
import_all_new = False
skip_all_new = False
count = 0
for index, src_file in enumerate(files):
if progress.wasCanceled():
break
if os.path.splitext(src_file)[1] == ".png" and convert_png and src_file not in SKIP_CONV:
tmp_src_file = os.path.join(tmp, os.path.basename(src_file))
tmp_src_file = os.path.splitext(tmp_src_file)[0] + ".gim"
quantize = QuantizeType.auto
for regex, q in FORCE_QUANTIZE:
if not regex.search(src_file) == None:
quantize = q
break
_CONV.png_to_gim(os.path.join(src, src_file), tmp_src_file, quantize)
src_file = tmp_src_file
else:
src_file = os.path.join(src, src_file)
for file in affected_files[index]:
if progress.wasCanceled():
break
dst_file = os.path.join(dst, file)
count += 1
# if count % 25 == 0:
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Importing...\n" + file)
progress.setValue(count)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
# We may be allowed to import files that don't exist, but we're
# going to ask them about it anyway.
if not os.path.isfile(dst_file):
if skip_all_new:
continue
if not import_all_new:
answer = QtGui.QMessageBox.question(
parent,
"File Not Found",
"File:\n\n" + file + "\n\n" +
"does not exist in the target directory. Import anyway?",
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.YesToAll | QtGui.QMessageBox.No | QtGui.QMessageBox.NoToAll,
defaultButton = QtGui.QMessageBox.No
)
if answer == QtGui.QMessageBox.YesToAll:
import_all_new = True
skip_all_new = False
elif answer == QtGui.QMessageBox.NoToAll:
skip_all_new = True
import_all_new = False
continue
elif answer == QtGui.QMessageBox.No:
continue
basedir = os.path.dirname(dst_file)
if not os.path.isdir(basedir):
os.makedirs(basedir)
shutil.copy2(src_file, dst_file)
shutil.rmtree(tmp)
progress.close()
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = 'http://osm.linfiniti.com/buildings-shp'
self.building_points_url = \
'http://osm.linfiniti.com/building-points-shp'
self.roads_url = 'http://osm.linfiniti.com/roads-shp'
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('InaSAFE OpenStreetMap Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if proxy is not None:
self.network_manager.setProxy(proxy)
self.restore_state()
# Setup the rectangle map tool
self.canvas = iface.mapCanvas()
self.rectangle_map_tool = \
RectangleMapTool(self.canvas)
self.rectangle_map_tool.rectangle_created.connect(
self.update_extent_from_rectangle)
self.button_extent_rectangle.clicked.connect(
self.drag_rectangle_on_map_canvas)
# Setup pan tool
self.pan_tool = QgsMapToolPan(self.canvas)
self.canvas.setMapTool(self.pan_tool)
self.update_extent_from_map_canvas()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:')
tips = m.BulletedList()
tips.add(
self.
tr('Your current extent, when opening this window, will be used to '
'determine the area for which you want data to be retrieved.'
'You can interactively select the area by using the '
'\'select on map\' button - which will temporarily hide this '
'window and allow you to drag a rectangle on the map. After you '
'have finished dragging the rectangle, this window will '
'reappear.'))
tips.add(
self.
tr('Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'))
tips.add(
self.
tr('By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'))
tips.add(
self.
tr('If a dataset already exists in the output directory it will be '
'overwritten.'))
tips.add(
self.
tr('This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(
m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self, extent):
"""Update extent value in GUI based from an extent.
:param extent: A list in the form [xmin, ymin, xmax, ymax] where all
coordinates provided are in Geographic / EPSG:4326.
:type extent: list
"""
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
def update_extent_from_map_canvas(self):
"""Update extent value in GUI based from value in map.
.. note:: Delegates to update_extent()
"""
self.groupBox.setTitle(self.tr('Bounding box from the map canvas'))
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.update_extent(extent)
def update_extent_from_rectangle(self):
"""Update extent value in GUI based from the QgsMapTool rectangle.
.. note:: Delegates to update_extent()
"""
self.show()
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.canvas.setMapTool(self.pan_tool)
rectangle = self.rectangle_map_tool.rectangle()
if rectangle:
self.groupBox.setTitle(self.tr('Bounding box from rectangle'))
extent = rectangle_geo_array(rectangle, self.iface.mapCanvas())
self.update_extent(extent)
def validate_extent(self):
"""Validate the bounding box before user click OK to download.
:return: True if the bounding box is valid, otherwise False
:rtype: bool
"""
min_latitude = float(str(self.min_latitude.text()))
max_latitude = float(str(self.max_latitude.text()))
min_longitude = float(str(self.min_longitude.text()))
max_longitude = float(str(self.max_longitude.text()))
# min_latitude < max_latitude
if min_latitude >= max_latitude:
return False
# min_longitude < max_longitude
if min_longitude >= max_longitude:
return False
# -90 <= latitude <= 90
if min_latitude < -90 or min_latitude > 90:
return False
if max_latitude < -90 or max_latitude > 90:
return False
# -180 <= longitude <= 180
if min_longitude < -180 or min_longitude > 180:
return False
if max_longitude < -180 or max_longitude > 180:
return False
return True
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory."""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(
QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def drag_rectangle_on_map_canvas(self):
"""Hide the dialog and allow the user to draw a rectangle."""
self.hide()
self.rectangle_map_tool.reset()
self.canvas.unsetMapTool(self.pan_tool)
self.canvas.setMapTool(self.rectangle_map_tool)
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Lock the groupbox
self.groupBox.setDisabled(True)
# Validate extent
valid_flag = self.validate_extent()
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the groupbox
self.groupBox.setEnabled(True)
return
# Get all the feature types
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads', 'building-points']
elif index == 1:
feature_types = ['buildings']
elif index == 2:
feature_types = ['building-points']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_checkBox.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, feature_type, overwrite)
self.download(feature_type, output_base_file_path)
try:
self.load_shapefile(feature_type, output_base_file_path)
except FileMissingError as exception:
display_warning_message_box(self, error_dialog_title,
exception.message)
self.done(QDialog.Accepted)
self.rectangle_map_tool.reset()
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title,
exception.message)
self.progress_dialog.cancel()
finally:
# Unlock the groupbox
self.groupBox.setEnabled(True)
def get_output_base_path(self, output_directory, output_prefix,
feature_type, overwrite):
"""Get a full base name path to save the shapefile.
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
path = os.path.join(output_directory,
'%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix('%s.shp' % path,
separator)
if suffix:
path = os.path.join(
output_directory, '%s%s%s%s' %
(output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return OsmDownloaderDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(self, title, question,
QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type, output_base_path):
"""Download shapefiles from Kartoza server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param output_base_path: The base path of the shape file.
:type output_base_path: str
:raises: ImportDialogError, CanceledImportDialogError
"""
# preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = ('{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude)
if feature_type == 'buildings':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.buildings_url, box=box)
elif feature_type == 'building-points':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.building_points_url, box=box)
else:
url = '{url}?bbox={box}&qgis_version=2'.format(url=self.roads_url,
box=box)
if 'LANG' in os.environ:
env_lang = os.environ['LANG']
url += '&lang=%s' % env_lang
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path, feature_type)
self.extract_zip(path, output_base_path)
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path, feature_type):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
# Infinite progress bar when the server is fetching data.
# The progress bar will be updated with the file size later.
self.progress_dialog.setMaximum(0)
self.progress_dialog.setMinimum(0)
self.progress_dialog.setValue(0)
# Get a pretty label from feature_type, but not translatable
label_feature_type = feature_type.replace('-', ' ')
label_text = self.tr('Fetching %s' % label_feature_type)
self.progress_dialog.setLabelText(label_text)
# Download Process
downloader = FileDownloader(self.network_manager, url, output_path,
self.progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
if result[0] == QNetworkReply.OperationCanceledError:
raise CanceledImportDialogError(error_message)
else:
raise DownloadError(error_message)
@staticmethod
def extract_zip(zip_path, destination_base_path):
"""Extract different extensions to the destination base path.
Example : test.zip contains a.shp, a.dbf, a.prj
and destination_base_path = '/tmp/CT-buildings
Expected result :
- /tmp/CT-buildings.shp
- /tmp/CT-buildings.dbf
- /tmp/CT-buildings.prj
If two files in the zip with the same extension, only one will be
copied.
:param zip_path: The path of the .zip file
:type zip_path: str
:param destination_base_path: The destination base path where the shp
will be written to.
:type destination_base_path: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
handle = open(zip_path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
extension = os.path.splitext(name)[1]
output_final_path = u'%s%s' % (destination_base_path, extension)
output_file = open(output_final_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers'))
def reject(self):
"""Redefinition of the reject() method
to remove the rectangle selection tool.
It will call the super method.
"""
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.rectangle_map_tool.reset()
super(OsmDownloaderDialog, self).reject()
class PetaJakartaDialog(QDialog, FORM_CLASS):
"""Downloader for petajakarta data.
.. versionadded: 3.3
"""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
.. versionadded: 3.3
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('PetaJakarta Downloader'))
self.iface = iface
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('PetaJakarta Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up things for context help
self.help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
# Allow toggling the help button
self.help_button.setCheckable(True)
self.help_button.toggled.connect(self.help_toggled)
self.main_stacked_widget.setCurrentIndex(1)
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
self.time_stamp = None
self.restore_state()
@pyqtSlot()
@pyqtSignature('bool') # prevents actions being handled twice
def help_toggled(self, flag):
"""Show or hide the help tab in the stacked widget.
.. versionadded: 3.3
:param flag: Flag indicating whether help should be shown or hidden.
:type flag: bool
"""
if flag:
self.help_button.setText(self.tr('Hide Help'))
self.show_help()
else:
self.help_button.setText(self.tr('Show Help'))
self.hide_help()
def hide_help(self):
"""Hide the usage info from the user.
.. versionadded:: 3.3
"""
self.main_stacked_widget.setCurrentIndex(1)
def show_help(self):
"""Show usage info to the user.
.. versionadded: 3.3
"""
# Read the header and footer html snippets
self.main_stacked_widget.setCurrentIndex(0)
header = html_header()
footer = html_footer()
string = header
message = peta_jakarta_help()
string += message.to_html()
string += footer
self.help_web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory.
.. versionadded: 3.3
"""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(
QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def accept(self):
"""Do PetaJakarta download and display it in QGIS.
.. versionadded: 3.3
"""
self.save_state()
try:
self.require_directory()
except CanceledImportDialogError:
return
QtGui.qApp.setOverrideCursor(QtGui.QCursor(QtCore.Qt.WaitCursor))
source = ('https://rem.petajakarta.org/banjir/data/api/v2/rem/flooded')
layer = QgsVectorLayer(source, 'flood', 'ogr', False)
self.time_stamp = time.strftime('%d-%b-%Y %H:%M:%S')
# Now save as shp
name = 'jakarta_flood.shp'
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_flag.isChecked()
date_stamp_flag = self.include_date_flag.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, date_stamp_flag, name, overwrite)
QgsVectorFileWriter.writeAsVectorFormat(layer, output_base_file_path,
'CP1250', None,
'ESRI Shapefile')
# Get rid of the GeoJSON layer and rather use local shp
del layer
self.copy_style(output_base_file_path)
self.copy_keywords(output_base_file_path)
layer = self.add_flooded_field(output_base_file_path)
# add the layer to the map
registry = QgsMapLayerRegistry.instance()
registry.addMapLayer(layer)
self.disable_busy_cursor()
self.done(QDialog.Accepted)
def add_flooded_field(self, shapefile_path):
"""Create the layer from the local shp adding the flooded field.
.. versionadded:: 3.3
Use this method to add a calculated field to a shapefile. The shapefile
should have a field called 'count' containing the number of flood
reports for the field. The field values will be set to 0 if the count
field is < 1, otherwise it will be set to 1.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
:return: A vector layer with the flooded field added.
:rtype: QgsVectorLayer
"""
layer = QgsVectorLayer(shapefile_path, self.tr('Jakarta Floods'),
'ogr')
# Add a calculated field indicating if a poly is flooded or not
# from PyQt4.QtCore import QVariant
layer.startEditing()
# Add field with integer from 0 to 4 which represents the flood
# class. Its the same as 'state' field except that is being treated
# as a string.
# This is used for cartography
flood_class_field = QgsField('floodclass', QVariant.Int)
layer.dataProvider().addAttributes([flood_class_field])
layer.commitChanges()
layer.startEditing()
flood_class_idx = layer.fieldNameIndex('floodclass')
flood_class_expression = QgsExpression('to_int(state)')
flood_class_expression.prepare(layer.pendingFields())
# Add field with boolean flag to say if the area is flooded
# This is used by the impact function
flooded_field = QgsField('flooded', QVariant.Int)
layer.dataProvider().addAttributes([flooded_field])
layer.commitChanges()
layer.startEditing()
flooded_idx = layer.fieldNameIndex('flooded')
flood_flag_expression = QgsExpression('state > 0')
flood_flag_expression.prepare(layer.pendingFields())
for feature in layer.getFeatures():
feature[flood_class_idx] = flood_class_expression.evaluate(feature)
feature[flooded_idx] = flood_flag_expression.evaluate(feature)
layer.updateFeature(feature)
layer.commitChanges()
return layer
def copy_keywords(self, shapefile_path):
"""Copy keywords from the OSM resource directory to the output path.
.. versionadded: 3.3
In addition to copying the template, tokens within the template will
be replaced with new values for the date token and title token.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
"""
source_xml_path = resources_path('petajakarta', 'flood-keywords.xml')
output_xml_path = shapefile_path.replace('shp', 'xml')
LOGGER.info('Copying xml to: %s' % output_xml_path)
title_token = '[TITLE]'
new_title = self.tr('Jakarta Floods - %s' % self.time_stamp)
date_token = '[DATE]'
new_date = self.time_stamp
with open(source_xml_path) as source_file, \
open(output_xml_path, 'w') as output_file:
for line in source_file:
line = line.replace(date_token, new_date)
line = line.replace(title_token, new_title)
output_file.write(line)
@staticmethod
def copy_style(shapefile_path):
"""Copy style from the OSM resource directory to the output path.
.. versionadded: 3.3
:param shapefile_path: Path to the shapefile that should get the path
added.
:type shapefile_path: basestring
"""
source_qml_path = resources_path('petajakarta', 'flood-style.qml')
output_qml_path = shapefile_path.replace('shp', 'qml')
LOGGER.info('Copying qml to: %s' % output_qml_path)
copy(source_qml_path, output_qml_path)
@staticmethod
def disable_busy_cursor():
"""Disable the hourglass cursor.
TODO: this is duplicated from dock.py
"""
while QtGui.qApp.overrideCursor() is not None and \
QtGui.qApp.overrideCursor().shape() == QtCore.Qt.WaitCursor:
QtGui.qApp.restoreOverrideCursor()
def get_output_base_path(self, output_directory, output_prefix,
with_date_stamp, feature_type, overwrite):
"""Get a full base name path to save the shapefile.
TODO: This is cut & paste from OSM - refactor to have one method
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param with_date_stamp: Whether to add a datestamp in between the
file prefix and the feature_type for the shapefile name.
:type output_prefix: str
:param feature_type: What kind of data will be downloaded. Will be
used for the shapefile name.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
if with_date_stamp and self.time_stamp is not None:
time_stamp = self.time_stamp.replace(' ', '-')
time_stamp = time_stamp.replace(':', '-')
time_stamp += '-'
feature_type = time_stamp + feature_type
path = os.path.join(output_directory,
'%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix('%s.shp' % path,
separator)
if suffix:
path = os.path.join(
output_directory, '%s%s%s%s' %
(output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
TODO: This is cut & paste from OSM - refactor to have one method
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return PetaJakartaDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
TODO: This is cut & paste from OSM - refactor to have one method
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(self, title, question,
QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
TODO: This is cut & paste from OSM - refactor to have one method
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.iface, self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers'))
def reject(self):
"""Redefinition of the reject() method.
It will call the super method.
"""
# add our own logic here...
super(PetaJakartaDialog, self).reject()
def setup_workspace(self):
umdimage = os.path.join(self.iso_dir, UMDIMAGE_DAT)
umdimage2 = os.path.join(self.iso_dir, UMDIMAGE2_DAT)
voice = os.path.join(self.iso_dir, VOICE_PAK)
self.generate_directories()
progress = QProgressDialog("", QtCore.QString(), 0, 11000, self)
progress.setWindowTitle("Setting up workspace...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setMinimumDuration(0)
progress.setValue(0)
progress.setAutoClose(False)
progress.setAutoReset(False)
progress.setLabelText("Creating directories...")
# Do the easy stuff first.
if not os.path.isdir(self.changes_dir):
os.makedirs(self.changes_dir)
progress.setValue(progress.value() + 1)
if not os.path.isdir(self.backup_dir):
os.makedirs(self.backup_dir)
progress.setValue(progress.value() + 1)
if not os.path.isdir(self.editor_data_dir):
os.makedirs(self.editor_data_dir)
progress.setValue(progress.value() + 1)
thread_fns = [
{"target": extract_umdimage, "kwargs": {"filename": umdimage, "out_dir": self.umdimage_dir, "eboot": self.eboot_path, "umdimage": UMDIMAGES.umdimage}},
{"target": extract_umdimage, "kwargs": {"filename": umdimage2, "out_dir": self.umdimage2_dir, "eboot": self.eboot_path, "umdimage": UMDIMAGES.umdimage2}},
{"target": extract_pak, "kwargs": {"filename": voice, "out_dir": self.voice_dir}},
]
# Going to capture stdout because I don't feel like
# rewriting the extract functions to play nice with GUI.
stdout = sys.stdout
sys.stdout = cStringIO.StringIO()
for thread_fn in thread_fns:
thread = threading.Thread(**thread_fn)
thread.start()
while thread.isAlive():
thread.join(THREAD_TIMEOUT)
output = [line for line in sys.stdout.getvalue().split('\n') if len(line) > 0]
progress.setValue(progress.value() + len(output))
if len(output) > 0:
progress.setLabelText("Extracting %s..." % output[-1])
sys.stdout = cStringIO.StringIO()
sys.stdout = stdout
# Give us an ISO directory for the editor to place modified files in.
progress.setLabelText("Copying ISO files...")
# ISO directory needs to not exist for copytree.
if os.path.isdir(self.edited_iso_dir):
shutil.rmtree(self.edited_iso_dir)
# One more thing we want threaded so it doesn't lock up the GUI.
thread = threading.Thread(target = shutil.copytree, kwargs = {"src": self.iso_dir, "dst": self.edited_iso_dir})
thread.start()
while thread.isAlive():
thread.join(THREAD_TIMEOUT)
progress.setLabelText("Copying ISO files...")
# It has to increase by some amount or it won't update and the UI will lock up.
progress.setValue(progress.value() + 1)
# shutil.copytree(self.iso_dir, self.edited_iso_dir)
progress.setValue(progress.value() + 1)
# Files we want to make blank, because they're unnecessary.
blank_files = [
os.path.join(self.edited_iso_dir, "PSP_GAME", "INSDIR", "UMDIMAGE.DAT"),
os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "DATA.BIN"),
os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "EBOOT.BIN"),
os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "PARAM.SFO"),
]
for blank in blank_files:
with open(blank, "wb") as f:
pass
# Copy the decrypted EBOOT into the ISO folder and apply our hacks to it.
progress.setLabelText("Hacking EBOOT...")
progress.setValue(progress.value() + 1)
hacked_eboot = BitStream(filename = self.eboot_path)
hacked_eboot, offset = apply_eboot_patches(hacked_eboot)
with open(os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"), "wb") as f:
hacked_eboot.tofile(f)
# shutil.copy(self.eboot_path, os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"))
progress.setLabelText("Extracting editor data...")
progress.setValue(progress.value() + 1)
# Extract the editor data.
editor_data = zipfile.ZipFile("data/editor_data.zip", "r")
editor_data.extractall(self.editor_data_dir)
editor_data.close()
progress.setValue(progress.maximum())
progress.close()
self.ui.grpStep4.setEnabled(False)
self.ui.grpStep5.setEnabled(True)
class SymbologySharingDialog(QtGui.QDialog, FORM_CLASS):
TAB_ALL = 0
TAB_INSTALLED = 1
TAB_SETTINGS = 2
def __init__(self, parent=None, iface=None):
"""Constructor.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
super(SymbologySharingDialog, self).__init__(parent)
self.setupUi(self)
self.iface = iface
self.repository_manager = RepositoryManager()
# Init the message bar
self.message_bar = QgsMessageBar(self)
self.message_bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.vlayoutRightColumn.insertWidget(0, self.message_bar)
# Mock plugin manager dialog
self.resize(796, 594)
self.setMinimumSize(QSize(790, 0))
self.setModal(True)
self.button_edit.setEnabled(False)
self.button_delete.setEnabled(False)
# Set QListWidgetItem
# All
icon_all = QIcon()
icon_all.addFile(resources_path('img', 'plugin.svg'), QSize(),
QIcon.Normal, QIcon.Off)
item_all = QListWidgetItem()
item_all.setIcon(icon_all)
item_all.setText(self.tr('All'))
# Installed
icon_installed = QIcon()
icon_installed.addFile(resources_path('img', 'plugin-installed.svg'),
QSize(), QIcon.Normal, QIcon.Off)
item_installed = QListWidgetItem()
item_installed.setIcon(icon_installed)
item_installed.setText(self.tr('Installed'))
item_all.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
# Settings
icon_settings = QIcon()
icon_settings.addFile(resources_path('img', 'settings.svg'), QSize(),
QIcon.Normal, QIcon.Off)
item_settings = QListWidgetItem()
item_settings.setIcon(icon_settings)
item_settings.setText(self.tr('Settings'))
item_all.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
# Add the list widget item to the widget
self.menu_list_widget.addItem(item_all)
self.menu_list_widget.addItem(item_installed)
self.menu_list_widget.addItem(item_settings)
# Slots
self.button_add.clicked.connect(self.add_repository)
self.button_edit.clicked.connect(self.edit_repository)
self.button_delete.clicked.connect(self.delete_repository)
self.menu_list_widget.currentRowChanged.connect(self.set_current_tab)
# Creating progress dialog for downloading stuffs
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('Symbology Sharing')
self.progress_dialog.setWindowTitle(title)
# Populate tree repositories with registered repositories
self.populate_tree_repositories()
def set_current_tab(self, index):
"""Set stacked widget based on active tab.
:param index: The index of the active list widget item.
:type index: int
"""
if index == (self.menu_list_widget.count() - 1):
# Switch to settings tab
self.stacked_menu_widget.setCurrentIndex(1)
else:
# Switch to plugins tab
self.stacked_menu_widget.setCurrentIndex(0)
def add_repository(self):
"""Open add repository dialog."""
dlg = ManageRepositoryDialog(self)
if not dlg.exec_():
return
for repo in self.repository_manager.repositories.values():
if dlg.line_edit_url.text().strip() == repo['url']:
self.message_bar.pushMessage(
self.tr(
'Unable to add another repository with the same URL!'),
QgsMessageBar.WARNING, 5)
return
repo_name = dlg.line_edit_name.text()
repo_url = dlg.line_edit_url.text().strip()
if repo_name in self.repository_manager.repositories:
repo_name += '(2)'
settings = QSettings()
settings.beginGroup(repo_settings_group())
settings.setValue(repo_name + '/url', repo_url)
# Fetch metadata
#TODO: Wrap RemoteRepository class into RepositoryManager
# This dialod will only need to call RepositoryManager.
# RepositoryManager will take care of the rest
remote_repository = RemoteRepository(repo_url)
remote_repository.fetch_metadata(self.progress_dialog)
# Show metadata
#TODO: Process this instead of showing it on message box :)
QMessageBox.information(self, self.tr("Test"),
remote_repository.metadata.data())
# Refresh tree repository
self.refresh_tree_repositories()
self.set_enabled_edit_delete_button(False)
def edit_repository(self):
"""Open edit repository dialog."""
selected_item = self.tree_repositories.currentItem()
if selected_item:
repo_name = selected_item.text(0)
if not repo_name:
return
dlg = ManageRepositoryDialog(self)
dlg.line_edit_name.setText(repo_name)
dlg.line_edit_url.setText(
self.repository_manager.repositories[repo_name]['url'])
if not dlg.exec_():
return
# Check if the changed URL is already there in the repo
new_url = dlg.line_edit_url.text().strip()
old_url = self.repository_manager.repositories[repo_name]['url']
for repo in self.repository_manager.repositories.values():
if new_url == repo['url'] and (old_url != new_url):
self.message_bar.pushMessage(
self.tr(
'Unable to add another repository with the same URL!'),
QgsMessageBar.WARNING, 5)
return
# Delete old repo and create a new entry
settings = QSettings()
settings.beginGroup(repo_settings_group())
settings.remove(repo_name)
new_name = dlg.line_edit_name.text()
if new_name in self.repository_manager.repositories and new_name != repo_name:
new_name += '(2)'
settings.setValue(new_name + '/url', new_url)
# Refresh tree repository
self.refresh_tree_repositories()
self.set_enabled_edit_delete_button(False)
def delete_repository(self):
"""Delete a repository in the tree widget."""
selected_item = self.tree_repositories.currentItem()
if selected_item:
repo_name = selected_item.text(0)
if not repo_name:
return
# Check if it's the official repository
settings = QSettings()
settings.beginGroup(repo_settings_group())
if settings.value(repo_name +
'/url') == self.repository_manager.OFFICIAL_REPO[1]:
self.message_bar.pushMessage(
self.tr('You can not remove the official repository!'),
QgsMessageBar.WARNING, 5)
return
warning = self.tr('Are you sure you want to remove the following '
'repository?') + '\n' + repo_name
if QMessageBox.warning(self, self.tr("QGIS Symbology Sharing"),
warning, QMessageBox.Yes,
QMessageBox.No) == QMessageBox.No:
return
settings.remove(repo_name)
# Refresh tree repository
self.refresh_tree_repositories()
self.set_enabled_edit_delete_button(False)
def refresh_tree_repositories(self):
"""Refresh tree repositories using new repositories data."""
self.repository_manager.load()
self.populate_tree_repositories()
def populate_tree_repositories(self):
"""Populate dictionary repositories to the tree widget."""
# Clear the current tree widget
self.tree_repositories.clear()
# Export the updated ones from the repository manager
for repo_name in self.repository_manager.repositories:
url = self.repository_manager.repositories[repo_name]['url']
item = QTreeWidgetItem(self.tree_repositories)
item.setText(0, repo_name)
item.setText(1, url)
self.tree_repositories.resizeColumnToContents(0)
self.tree_repositories.resizeColumnToContents(1)
self.tree_repositories.sortItems(1, Qt.AscendingOrder)
def on_tree_repositories_itemSelectionChanged(self):
"""Slot for when the itemSelectionChanged signal emitted."""
# Activate edit and delete button
self.set_enabled_edit_delete_button(True)
def set_enabled_edit_delete_button(self, is_enabled):
"""Disable edit and delete button.
:param is_enabled: Boolean is enabled or not.
:type is_enabled: bool
"""
self.button_edit.setEnabled(is_enabled)
self.button_delete.setEnabled(is_enabled)
class PetaBencanaDialog(QDialog, FORM_CLASS):
"""Downloader for PetaBencana data.
.. versionadded: 3.3
"""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
.. versionadded: 3.3
:param parent: Optional widget to use as parent.
:type parent: QWidget
:param iface: An instance of QGisInterface.
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
title = self.tr('PetaBencana Downloader')
self.setWindowTitle(title)
self.iface = iface
self.source = None
self.radio_button_group = QButtonGroup()
self.radio_button_group.addButton(self.radio_button_production)
self.radio_button_group.addButton(self.radio_button_development)
self.radio_button_group.setExclusive(True)
self.radio_button_production.setChecked(True)
self.populate_combo_box()
developer_mode = setting('developer_mode', False, bool)
if not developer_mode:
self.radio_button_widget.hide()
self.source_label.hide()
self.output_group.adjustSize()
# signals
self.radio_button_production.clicked.connect(self.populate_combo_box)
self.radio_button_development.clicked.connect(self.populate_combo_box)
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
self.progress_dialog.setWindowTitle(title)
# Set up things for context help
self.help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
# Allow toggling the help button
self.help_button.setCheckable(True)
self.help_button.toggled.connect(self.help_toggled)
self.main_stacked_widget.setCurrentIndex(1)
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
self.time_stamp = None
self.restore_state()
@pyqtSlot()
@pyqtSignature('bool') # prevents actions being handled twice
def help_toggled(self, flag):
"""Show or hide the help tab in the stacked widget.
.. versionadded: 3.3
:param flag: Flag indicating whether help should be shown or hidden.
:type flag: bool
"""
if flag:
self.help_button.setText(self.tr('Hide Help'))
self.show_help()
else:
self.help_button.setText(self.tr('Show Help'))
self.hide_help()
def hide_help(self):
"""Hide the usage info from the user.
.. versionadded:: 3.3
"""
self.main_stacked_widget.setCurrentIndex(1)
def show_help(self):
"""Show usage info to the user.
.. versionadded: 3.3
"""
# Read the header and footer html snippets
self.main_stacked_widget.setCurrentIndex(0)
header = html_header()
footer = html_footer()
string = header
message = peta_bencana_help()
string += message.to_html()
string += footer
self.help_web_view.setHtml(string)
def restore_state(self):
"""Read last state of GUI from configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
"""Store current state of GUI to configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory.
.. versionadded: 3.3
"""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(
QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def accept(self):
"""Do PetaBencana download and display it in QGIS.
.. versionadded: 3.3
"""
self.save_state()
try:
self.require_directory()
except CanceledImportDialogError:
return
QtGui.qApp.setOverrideCursor(QtGui.QCursor(QtCore.Qt.WaitCursor))
source = self.define_url()
# save the file as json first
name = 'jakarta_flood.json'
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_flag.isChecked()
date_stamp_flag = self.include_date_flag.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, date_stamp_flag, name, overwrite)
title = self.tr("Can't access API")
try:
self.download(source, output_base_file_path)
# Open downloaded file as QgsMapLayer
layer = QgsVectorLayer(output_base_file_path, 'flood', 'ogr',
False)
except Exception as e:
disable_busy_cursor()
QMessageBox.critical(self, title, str(e))
return
self.time_stamp = time.strftime('%d-%b-%Y %H:%M:%S')
# Now save as shp
name = 'jakarta_flood.shp'
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, date_stamp_flag, name, overwrite)
QgsVectorFileWriter.writeAsVectorFormat(layer, output_base_file_path,
'CP1250', None,
'ESRI Shapefile')
# Get rid of the GeoJSON layer and rather use local shp
del layer
self.copy_style(output_base_file_path)
self.copy_keywords(output_base_file_path)
layer = self.add_flooded_field(output_base_file_path)
# check if the layer has feature or not
if layer.featureCount() <= 0:
city = self.city_combo_box.currentText()
message = self.tr('There are no floods data available on {city} '
'at this time.').format(city=city)
display_warning_message_box(self, self.tr('No data'), message)
disable_busy_cursor()
else:
# add the layer to the map
registry = QgsMapLayerRegistry.instance()
registry.addMapLayer(layer)
disable_busy_cursor()
self.done(QDialog.Accepted)
def add_flooded_field(self, shapefile_path):
"""Create the layer from the local shp adding the flooded field.
.. versionadded:: 3.3
Use this method to add a calculated field to a shapefile. The shapefile
should have a field called 'count' containing the number of flood
reports for the field. The field values will be set to 0 if the count
field is < 1, otherwise it will be set to 1.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
:return: A vector layer with the flooded field added.
:rtype: QgsVectorLayer
"""
layer = QgsVectorLayer(shapefile_path, self.tr('Jakarta Floods'),
'ogr')
# Add a calculated field indicating if a poly is flooded or not
# from PyQt4.QtCore import QVariant
layer.startEditing()
# Add field with integer from 0 to 4 which represents the flood
# class. Its the same as 'state' field except that is being treated
# as a string.
# This is used for cartography
flood_class_field = QgsField('floodclass', QVariant.Int)
layer.dataProvider().addAttributes([flood_class_field])
layer.commitChanges()
layer.startEditing()
flood_class_idx = layer.fieldNameIndex('floodclass')
flood_class_expression = QgsExpression('to_int(state)')
context = QgsExpressionContext()
context.setFields(layer.pendingFields())
flood_class_expression.prepare(context)
# Add field with boolean flag to say if the area is flooded
# This is used by the impact function
flooded_field = QgsField('flooded', QVariant.Int)
layer.dataProvider().addAttributes([flooded_field])
layer.commitChanges()
layer.startEditing()
flooded_idx = layer.fieldNameIndex('flooded')
flood_flag_expression = QgsExpression('state > 0')
flood_flag_expression.prepare(context)
for feature in layer.getFeatures():
context.setFeature(feature)
feature[flood_class_idx] = flood_class_expression.evaluate(context)
feature[flooded_idx] = flood_flag_expression.evaluate(context)
layer.updateFeature(feature)
layer.commitChanges()
return layer
def copy_keywords(self, shapefile_path):
"""Copy keywords from the OSM resource directory to the output path.
.. versionadded: 3.3
In addition to copying the template, tokens within the template will
be replaced with new values for the date token and title token.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
"""
source_xml_path = resources_path('petabencana', 'flood-keywords.xml')
output_xml_path = shapefile_path.replace('shp', 'xml')
LOGGER.info('Copying xml to: %s' % output_xml_path)
title_token = '[TITLE]'
new_title = self.tr('Jakarta Floods - %s' % self.time_stamp)
date_token = '[DATE]'
new_date = self.time_stamp
with open(source_xml_path) as source_file, \
open(output_xml_path, 'w') as output_file:
for line in source_file:
line = line.replace(date_token, new_date)
line = line.replace(title_token, new_title)
output_file.write(line)
@staticmethod
def copy_style(shapefile_path):
"""Copy style from the OSM resource directory to the output path.
.. versionadded: 3.3
:param shapefile_path: Path to the shapefile that should get the path
added.
:type shapefile_path: basestring
"""
source_qml_path = resources_path('petabencana', 'flood-style.qml')
output_qml_path = shapefile_path.replace('shp', 'qml')
LOGGER.info('Copying qml to: %s' % output_qml_path)
copy(source_qml_path, output_qml_path)
def get_output_base_path(self, output_directory, output_prefix,
with_date_stamp, feature_type, overwrite):
"""Get a full base name path to save the shapefile.
TODO: This is cut & paste from OSM - refactor to have one method
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param with_date_stamp: Whether to add a datestamp in between the
file prefix and the feature_type for the shapefile name.
:type output_prefix: str
:param feature_type: What kind of data will be downloaded. Will be
used for the shapefile name.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
if with_date_stamp and self.time_stamp is not None:
time_stamp = self.time_stamp.replace(' ', '-')
time_stamp = time_stamp.replace(':', '-')
time_stamp += '-'
feature_type = time_stamp + feature_type
path = os.path.join(output_directory,
'%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix('%s.shp' % path,
separator)
if suffix:
path = os.path.join(
output_directory, '%s%s%s%s' %
(output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
TODO: This is cut & paste from OSM - refactor to have one method
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return PetaBencanaDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
TODO: This is cut & paste from OSM - refactor to have one method
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(self, title, question,
QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
TODO: This is cut & paste from OSM - refactor to have one method
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapSettings().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.iface, self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers'))
def reject(self):
"""Redefinition of the method.
It will call the super method.
"""
super(PetaBencanaDialog, self).reject()
def download(self, url, output_path):
"""Download file from API url and write to output path.
:param url: URL of the API.
:type url: str
:param output_path: Path of output file,
:type output_path: str
"""
request_failed_message = self.tr(
"Can't access PetaBencana API: {source}").format(source=url)
downloader = FileDownloader(url, output_path)
result, message = downloader.download()
if not result:
display_warning_message_box(
self, self.tr('Download error'),
self.tr(request_failed_message + '\n' + message))
if result == QNetworkReply.OperationCanceledError:
display_warning_message_box(self, self.tr('Download error'),
self.tr(message))
# The function below might be usefull for future usage.
# def get_available_area(self):
# """Function to automatically get the available area on API.
# *still cannot get string data from QByteArray*
# """
# available_area = []
# network_manager = QgsNetworkAccessManager.instance()
# api_url = QUrl('https://data.petabencana.id/cities')
# api_request = QNetworkRequest(api_url)
# api_response = network_manager.get(api_request)
# data = api_response.readAll()
# json_response = QScriptEngine().evaluate(data)
# geometries = json_response.property('output').property('geometries')
# iterator = QScriptValueIterator(geometries)
# while iterator.hasNext():
# iterator.next()
# geometry = iterator.value()
# geometry_code = (
# geometry.property('properties').property('code').toString())
# available_area.append(geometry_code)
def populate_combo_box(self):
"""Populate combobox for selecting city."""
if self.radio_button_production.isChecked():
self.source = production_api['url']
available_data = production_api['available_data']
else:
self.source = development_api['url']
available_data = development_api['available_data']
self.city_combo_box.clear()
for index, data in enumerate(available_data):
self.city_combo_box.addItem(data['name'])
self.city_combo_box.setItemData(index, data['code'], Qt.UserRole)
def define_url(self):
"""Define API url based on which source is selected.
:return: Valid url of selected source.
:rtype: str
"""
current_index = self.city_combo_box.currentIndex()
city_code = self.city_combo_box.itemData(current_index, Qt.UserRole)
source = (self.source).format(city_code=city_code)
return source
class CpkPacker():
def __init__(self, parent = None):
self.parent = parent
self.process = None
def __pack_cpk(self, csv, cpk):
self.progress.setValue(0)
self.progress.setMaximum(1000)
self.progress.setLabelText("Building %s" % cpk)
process = QProcess()
process.start("tools/cpkmakec", [csv, cpk, "-align=2048", "-mode=FILENAME"])
percent = 0
while not process.waitForFinished(100):
output = QString(process.readAll())
output = output.split("\n", QString.SkipEmptyParts)
for line in output:
line = common.qt_to_unicode(line)
match = OUTPUT_RE.search(line)
if match == None:
continue
percent = float(match.group(1)) * 1000
self.progress.setValue(percent)
percent += 1
def __cache_outdated(self, src_dir, cache_file):
if not os.path.isfile(cache_file):
return True
cache_updated = os.path.getmtime(cache_file)
for src_file in list_all_files(src_dir):
if os.path.getmtime(src_file) > cache_updated:
return True
return False
def create_archives(self):
try:
self.width = self.parent.width()
self.height = self.parent.height()
self.x = self.parent.x()
self.y = self.parent.y()
except:
self.width = 1920
self.height = 1080
self.x = 0
self.y = 0
self.progress = QProgressDialog("Reading...", QtCore.QString(), 0, 7600, self.parent)
self.progress.setWindowModality(Qt.Qt.WindowModal)
self.progress.setValue(0)
self.progress.setAutoClose(False)
self.progress.setMinimumDuration(0)
USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "USRDIR")
eboot_path = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN")
eboot = BitStream(filename = eboot_path)
eboot = eboot_patch.apply_eboot_patches(eboot)
# So we can loop. :)
ARCHIVE_INFO = [
{
"dir": common.editor_config.data00_dir,
"cpk": os.path.join(USRDIR, "data00.cpk"),
"csv": os.path.join("data", "data00.csv" if not common.editor_config.quick_build else "data00-quick.csv"),
"name": "data00.cpk",
"pack": common.editor_config.pack_data00,
},
{
"dir": common.editor_config.data01_dir,
"cpk": os.path.join(USRDIR, "data01.cpk"),
"csv": os.path.join("data", "data01.csv" if not common.editor_config.quick_build else "data01-quick.csv"),
"name": "data01.cpk",
"pack": common.editor_config.pack_data01,
},
]
# temp_dir = tempfile.mkdtemp(prefix = "sdse-")
temp_dir = common.editor_config.build_cache
for archive in ARCHIVE_INFO:
if not archive["pack"]:
continue
self.progress.setWindowTitle("Building " + archive["name"])
toc_info = {}
file_list = None
if archive["toc"]:
file_list = []
toc = get_toc(eboot, archive["toc"])
for entry in toc:
filename = entry["filename"]
pos_pos = entry["file_pos_pos"]
len_pos = entry["file_len_pos"]
toc_info[filename] = [pos_pos, len_pos]
file_list.append(filename)
# Causes memory issues if I use the original order, for whatever reason.
file_list = None
csv_template_f = open(archive["csv"], "rb")
csv_template = csv.reader(csv_template_f)
csv_out_path = os.path.join(temp_dir, "cpk.csv")
csv_out_f = open(csv_out_path, "wb")
csv_out = csv.writer(csv_out_f)
for row in csv_template:
if len(row) < 4:
continue
base_path = row[0]
real_path = os.path.join(archive["dir"], base_path)
out_path = os.path.join(temp_dir, archive["name"], base_path)
self.progress.setValue(self.progress.value() + 1)
self.progress.setLabelText("Reading...\n%s" % real_path)
# All items in the CPK list should be files.
# Therefore, if we have a directory, then it needs to be packed.
if os.path.isdir(real_path):
if self.__cache_outdated(real_path, out_path):
out_dir = os.path.dirname(out_path)
try:
os.makedirs(out_dir)
except:
pass
data = pack_dir(real_path)
with open(out_path, "wb") as out_file:
data.tofile(out_file)
del data
elif os.path.isfile(real_path):
# If it's a file, though, we can just use it directly.
out_path = real_path
row[0] = out_path
csv_out.writerow(row)
csv_template_f.close()
csv_out_f.close()
self.__pack_cpk(csv_out_path, archive["cpk"])
# We're playing fast and loose with the file count anyway, so why not?
self.file_count += 1
self.progress.setValue(self.file_count)
self.progress.setLabelText("Saving " + archive["name"] + "...")
if archive["toc"]:
for entry in table_of_contents:
if not entry in toc_info:
_LOGGER.warning("%s missing from %s table of contents." % (entry, archive["name"]))
continue
file_pos = table_of_contents[entry]["pos"]
file_size = table_of_contents[entry]["size"]
eboot.overwrite(BitStream(uintle = file_pos, length = 32), toc_info[entry][0] * 8)
eboot.overwrite(BitStream(uintle = file_size, length = 32), toc_info[entry][1] * 8)
del table_of_contents
self.progress.setWindowTitle("Building...")
self.progress.setLabelText("Saving EBOOT.BIN...")
self.progress.setValue(self.progress.maximum())
with open(eboot_path, "wb") as f:
eboot.tofile(f)
# Text replacement
to_replace = eboot_text.get_eboot_text()
for replacement in to_replace:
orig = bytearray(replacement.orig, encoding = replacement.enc)
# If they left something blank, write the original text back.
if len(replacement.text) == 0:
data = orig
else:
data = bytearray(replacement.text, encoding = replacement.enc)
pos = replacement.pos.int + eboot_offset
padding = len(orig) - len(data)
if padding > 0:
# Null bytes to fill the rest of the space the original took.
data.extend(bytearray(padding))
data = ConstBitStream(bytes = data)
eboot.overwrite(data, pos * 8)
eboot_out = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN")
with open(eboot_out, "wb") as f:
eboot.tofile(f)
self.progress.close()
def export_umdimage(src, dst, convert_gim=True, unique=False, parent=None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError(
"Cannot export %s. Source and destination directories are the same."
% src)
answer = QtGui.QMessageBox.question(
parent,
"Export Directory",
"Exporting directory:\n\n" + src + "\n\n" + "into directory:\n\n" +
dst + "\n\n" + "Proceed?",
buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton=QtGui.QMessageBox.No)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent)
progress.setWindowTitle("Exporting...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
if parent:
width = parent.width()
height = parent.height()
x = parent.x()
y = parent.y()
else:
width = 1920
height = 1080
x = 0
y = 0
seen_groups = []
count = 0
last_update = time.time()
progress.setMaximum(60000)
for filename in list_all_files(src):
if progress.wasCanceled():
break
count += 1
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Exporting...\n" + filename)
progress.setValue(count)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
base_name = filename[len(src) + 1:]
if unique:
dupe_name = os.path.join("umdimage", base_name)
dupe_name = os.path.normpath(os.path.normcase(dupe_name))
group = _DUPE_DB.group_from_file(dupe_name)
if group in seen_groups:
continue
if not group == None:
seen_groups.append(group)
dst_file = os.path.join(dst, base_name)
dst_dir = os.path.dirname(dst_file)
ext = os.path.splitext(dst_file)[1].lower()
try:
os.makedirs(dst_dir)
except:
pass
if ext == ".gim" and convert_gim:
dst_file = os.path.splitext(dst_file)[0] + ".png"
_CONV.gim_to_png(filename, dst_file)
else:
shutil.copy2(filename, dst_file)
progress.close()
class OsmDownloader(QDialog, Ui_OsmDownloaderBase):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = "http://osm.linfiniti.com/buildings-shp"
self.roads_url = "http://osm.linfiniti.com/roads-shp"
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr("InaSAFE OpenStreetMap Downloader")
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if not proxy is None:
self.network_manager.setProxy(proxy)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent will be used to determine the area for which '
'you want data to be retrieved. You can adjust it manually using '
'the bounding box options below.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self):
""" Update extent value in GUI based from value in map."""
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads']
elif index == 1:
feature_types = ['buildings']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
self.download(feature_type)
self.load_shapefile(feature_type)
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self,
self.tr("InaSAFE OpenStreetMap downloader error"),
str(myEx))
self.progress_dialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = str(self.output_directory.text())
if os.path.exists(path):
return
title = self.tr("Directory %s not exist") % path
question = self.tr(
"Directory %s not exist. Do you want to create it?"
) % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title,
question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker, PyArgumentList
QMessageBox.warning(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type):
"""Download shapefiles from Linfinti server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError, CanceledImportDialogError
"""
## preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = (
'{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(
min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude
)
output_prefix = self.filename_prefix.text()
if feature_type == 'buildings':
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.buildings_url, box=box)
else:
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.roads_url, box=box)
if output_prefix is not None:
url += '&output_prefix=%s' % output_prefix
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path)
#print path
#print str(self.output_directory.text())
self.extract_zip(path, str(self.output_directory.text()))
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
self.progress_dialog.setMaximum(100)
self.progress_dialog.setValue(0)
# label_text = "Begin downloading shapefile from " \
# + "%s ..."
# self.progress_dialog.setLabelText(self.tr(label_text) % (url))
label_text = self.tr("Downloading shapefile")
self.progress_dialog.setLabelText(label_text)
result = download_url(
self.network_manager, url, output_path,
self.progress_dialog)
if result[0] is not True:
_, error_message = result
raise ImportDialogError(error_message)
@staticmethod
def extract_zip(path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
handle = open(path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
output_path = os.path.join(output_dir, name)
output_file = open(output_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type):
"""
Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when buildings.shp not exist
"""
output_prefix = self.filename_prefix.text()
path = str(self.output_directory.text())
path = os.path.join(path, '%s%s.shp' % (output_prefix, feature_type))
if not os.path.exists(path):
message = self.tr(
"%s don't exist. The server doesn't have any data."
)
raise ImportDialogError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
def import_umdimage(src, dst, convert_png=True, propogate=True, parent=None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError(
"Cannot import %s. Source and destination directories are the same."
% src)
answer = QtGui.QMessageBox.question(
parent,
"Import Directory",
"Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" +
dst + "\n\n" + "Any affected files will be backed up. Proceed?",
buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton=QtGui.QMessageBox.No)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Finding files...", "Cancel", 0, 1, parent)
progress.setWindowTitle("Importing...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
if parent:
width = parent.width()
height = parent.height()
x = parent.x()
y = parent.y()
else:
width = 1920
height = 1080
x = 0
y = 0
progress.setMaximum(0)
progress.setValue(0)
# The raw list of files we're importing.
files = []
# A list of lists, including all dupes of the files being imported, too.
affected_files = []
file_count = 0
dupe_base = "umdimage"
tmp = tempfile.mkdtemp(prefix="sdse-")
seen_groups = []
count = 0
last_update = time.time()
for file in list_all_files(src):
if progress.wasCanceled():
break
# Strip our base directory, so we have just a relative file list.
file = os.path.normpath(os.path.normcase(file[len(src) + 1:]))
files.append(file)
count += 1
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Finding files...\n" + file)
# progress.setValue(count)
progress.setValue(progress.value() ^ 1)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
affected_files.append([])
if os.path.splitext(
file)[1] == ".png" and convert_png and file not in SKIP_CONV:
file = os.path.splitext(file)[0] + ".gim"
if propogate:
file_group = _DUPE_DB.group_from_file(os.path.join(
dupe_base, file))
else:
file_group = None
if file_group in seen_groups:
continue
# If there are no dupes, just add this file.
if file_group == None:
affected_files[-1].append(file)
file_count += 1
continue
seen_groups.append(file_group)
for dupe in _DUPE_DB.files_in_group(file_group):
# Minus the "umdimage" part
dupe = dupe[len(dupe_base) + 1:]
affected_files[-1].append(dupe)
file_count += 1
progress.setValue(0)
progress.setMaximum(file_count)
# Make a backup first.
backup_dir = None
count = 0
for file_set in affected_files:
if progress.wasCanceled():
break
for file in file_set:
if progress.wasCanceled():
break
count += 1
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Backing up...\n" + file)
progress.setValue(count)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
# It's perfectly possible we want to import some files that
# don't already exist. Such as when importing a directory
# with added lines.
if not os.path.isfile(os.path.join(dst, file)):
continue
backup_dir = backup_files(dst, [file],
suffix="_IMPORT",
backup_dir=backup_dir)
progress.setValue(0)
# And now do our importing.
import_all_new = False
skip_all_new = False
count = 0
for index, src_file in enumerate(files):
if progress.wasCanceled():
break
if os.path.splitext(src_file)[
1] == ".png" and convert_png and src_file not in SKIP_CONV:
tmp_src_file = os.path.join(tmp, os.path.basename(src_file))
tmp_src_file = os.path.splitext(tmp_src_file)[0] + ".gim"
quantize = QuantizeType.auto
for regex, q in FORCE_QUANTIZE:
if not regex.search(src_file) == None:
quantize = q
break
_CONV.png_to_gim(os.path.join(src, src_file), tmp_src_file,
quantize)
src_file = tmp_src_file
else:
src_file = os.path.join(src, src_file)
for file in affected_files[index]:
if progress.wasCanceled():
break
dst_file = os.path.join(dst, file)
count += 1
# if count % 25 == 0:
if time.time() - last_update > MIN_INTERVAL or count % 25 == 0:
last_update = time.time()
progress.setLabelText("Importing...\n" + file)
progress.setValue(count)
# Re-center the dialog.
progress_w = progress.geometry().width()
progress_h = progress.geometry().height()
new_x = x + ((width - progress_w) / 2)
new_y = y + ((height - progress_h) / 2)
progress.move(new_x, new_y)
# We may be allowed to import files that don't exist, but we're
# going to ask them about it anyway.
if not os.path.isfile(dst_file):
if skip_all_new:
continue
if not import_all_new:
answer = QtGui.QMessageBox.question(
parent,
"File Not Found",
"File:\n\n" + file + "\n\n" +
"does not exist in the target directory. Import anyway?",
buttons=QtGui.QMessageBox.Yes
| QtGui.QMessageBox.YesToAll | QtGui.QMessageBox.No
| QtGui.QMessageBox.NoToAll,
defaultButton=QtGui.QMessageBox.No)
if answer == QtGui.QMessageBox.YesToAll:
import_all_new = True
skip_all_new = False
elif answer == QtGui.QMessageBox.NoToAll:
skip_all_new = True
import_all_new = False
continue
elif answer == QtGui.QMessageBox.No:
continue
basedir = os.path.dirname(dst_file)
if not os.path.isdir(basedir):
os.makedirs(basedir)
shutil.copy2(src_file, dst_file)
shutil.rmtree(tmp)
progress.close()
class OsmDownloader(QDialog, Ui_OsmDownloaderBase):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.url = "http://osm.linfiniti.com/buildings-shp"
# creating progress dialog for download
self.progressDialog = QProgressDialog(self)
self.progressDialog.setAutoClose(False)
myTitle = self.tr("InaSAFE OpenStreetMap Downloader")
self.progressDialog.setWindowTitle(myTitle)
# Set up context help
helpButton = self.buttonBox.button(QtGui.QDialogButtonBox.Help)
QtCore.QObject.connect(helpButton, QtCore.SIGNAL('clicked()'),
self.show_help)
self.show_info()
self.network_manager = QNetworkAccessManager(self)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') data from the '
'OpenStreetMap project for you. The downloaded data will have '
'InaSAFE keywords defined and a default QGIS style applied. To '
'use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Use QGIS to zoom in to the area for which you want building data '
'to be retrieved.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called buildings.shp (and its associated files).'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.webView.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
mySetting = QSettings()
self.outDir.setText(mySetting.value('directory'))
def save_state(self):
""" Store current state of GUI to configuration file """
mySetting = QSettings()
mySetting.setValue('directory', self.outDir.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help('openstreetmap_downloader')
def update_extent(self):
""" Update extent value in GUI based from value in map."""
myExtent = self.iface.mapCanvas().extent()
self.minLongitude.setText(str(myExtent.xMinimum()))
self.minLatitude.setText(str(myExtent.yMinimum()))
self.maxLongitude.setText(str(myExtent.xMaximum()))
self.maxLatitude.setText(str(myExtent.yMaximum()))
@pyqtSignature('') # prevents actions being handled twice
def on_pBtnDir_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.outDir.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
try:
self.save_state()
self.require_directory()
self.download()
self.load_shapefile()
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self,
self.tr("InaSAFE OpenStreetMap downloader error"),
str(myEx))
self.progressDialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
myDir = str(self.outDir.text())
if os.path.exists(myDir):
return
myTitle = self.tr("Directory %s not exist") % myDir
myQuestion = self.tr(
"Directory %s not exist. Do you want to create it?"
) % myDir
# noinspection PyCallByClass,PyTypeChecker
myAnswer = QMessageBox.question(
self, myTitle,
myQuestion, QMessageBox.Yes | QMessageBox.No)
if myAnswer == QMessageBox.Yes:
os.makedirs(myDir)
else:
raise CanceledImportDialogError()
def download(self):
"""Download shapefiles from Linfinti server.
:raises: ImportDialogError, CanceledImportDialogError
"""
## preparing necessary data
myMinLng = str(self.minLongitude.text())
myMinLat = str(self.minLatitude.text())
myMaxLng = str(self.maxLongitude.text())
myMaxLat = str(self.maxLatitude.text())
myCoordinate = "{myMinLng},{myMinLat},{myMaxLng},{myMaxLat}".format(
myMinLng=myMinLng,
myMinLat=myMinLat,
myMaxLng=myMaxLng,
myMaxLat=myMaxLat
)
myShapeUrl = "{url}?bbox={myCoordinate}".format(
url=self.url,
myCoordinate=myCoordinate
)
myFilePath = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(myShapeUrl, myFilePath)
print myFilePath
print str(self.outDir.text())
self.extract_zip(myFilePath, str(self.outDir.text()))
self.progressDialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
self.progressDialog.show()
self.progressDialog.setMaximum(100)
self.progressDialog.setValue(0)
# myLabelText = "Begin downloading shapefile from " \
# + "%s ..."
# self.progressDialog.setLabelText(self.tr(myLabelText) % (url))
myLabelText = self.tr("Downloading shapefile")
self.progressDialog.setLabelText(myLabelText)
myResult = download_url(
self.network_manager, url, output_path,
self.progressDialog)
if myResult is not True:
_, myErrorMessage = myResult
raise ImportDialogError(myErrorMessage)
def extract_zip(self, path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
myHandle = open(path, 'rb')
myZip = zipfile.ZipFile(myHandle)
for myName in myZip.namelist():
myOutPath = os.path.join(output_dir, myName)
myOutFile = open(myOutPath, 'wb')
myOutFile.write(myZip.read(myName))
myOutFile.close()
myHandle.close()
def load_shapefile(self):
"""
Load downloaded shape file to QGIS Main Window.
:raises: ImportDialogError - when buildings.shp not exist
"""
myDir = str(self.outDir.text())
myPath = os.path.join(myDir, 'buildings.shp')
if not os.path.exists(myPath):
myMessage = self.tr(
"%s don't exist. The server don't have buildings data."
)
raise ImportDialogError(myMessage)
self.iface.addVectorLayer(myPath, 'buildings', 'ogr')
def import_umdimage2(src, dst, convert_png = True, propogate = True, parent = None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError("Cannot import %s. Source and destination directories are the same." % src)
answer = QtGui.QMessageBox.question(
parent,
"Import Directory",
"Importing directory:\n\n" + src + "\n\n" +
"into directory:\n\n" + dst + "\n\n" +
"Any affected files will be backed up. Proceed?",
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton = QtGui.QMessageBox.No
)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Importing...", "Cancel", 0, 0, parent)
progress.setWindowTitle("Importing...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
tmp_dst = tempfile.mkdtemp(prefix = "sdse-")
backup_dir = None
for pak in glob.iglob(os.path.join(src, "bg_*.pak")):
if progress.wasCanceled():
break
pak_name = os.path.basename(pak)
backup_dir = backup_files(dst, [pak_name], suffix = "_IMPORT", backup_dir = backup_dir)
# If we have a regular file with the bg_*.pak name, then just drop it in.
if os.path.isfile(pak):
progress.setLabelText("Importing:\n" + pak_name)
progress.setValue(progress.value() ^ 1)
shutil.copy2(pak, os.path.join(dst, pak_name))
# Otherwise, if it's a directory, insert all the textures we find
# into the target bg_*.pak file.
elif os.path.isdir(pak):
for image in list_all_files(pak):
if progress.wasCanceled():
break
ext = os.path.splitext(image)[1].lower()
if ext == ".png" and not convert_png:
continue
base_name = image[len(src) + 1:]
dst_files = []
if propogate:
dupe_name = os.path.splitext(base_name)[0] + ".gim"
dupe_name = os.path.join("umdimage2", dupe_name)
dupe_name = os.path.normpath(os.path.normcase(dupe_name))
dupes = _DUPE_DB.files_in_same_group(dupe_name)
if dupes == None:
dupes = [dupe_name]
for dupe in dupes:
dst_file = dupe[10:] # chop off the "umdimage2/"
dst_file = os.path.splitext(dst_file)[0] + ext # original extension
dst_file = os.path.join(tmp_dst, dst_file)
dst_files.append(dst_file)
else:
dst_files = [os.path.join(tmp_dst, base_name)]
for dst_file in dst_files:
try:
os.makedirs(os.path.dirname(dst_file))
except:
pass
shutil.copy(image, dst_file)
if progress.wasCanceled():
break
progress.setLabelText("Inserting textures into:\n" + pak_name)
progress.setValue(progress.value() ^ 1)
pak_dir = os.path.join(tmp_dst, pak_name)
pak_file = os.path.join(dst, pak_name)
# If we didn't copy anything over, just move on.
if not os.path.isdir(pak_dir):
continue
thread = threading.Thread(target = insert_textures, args = (pak_dir, pak_file))
thread.start()
while thread.isAlive():
thread.join(MIN_INTERVAL)
progress.setValue(progress.value() ^ 1)
if progress.wasCanceled():
progress.setLabelText("Canceling...")
shutil.rmtree(tmp_dst)
progress.close()
class specread:
def __init__(self, specfile, startLineNum=0, beamline='APS-15IDC',det='CCD',data={},par={}):
self.Data=data
self.Par=par
self.specfile=specfile
if beamline=='APS-15IDC':
self.APS_15IDC(startLineNum=startLineNum,det=det)
if beamline=='APS-9IDC':
self.APS_9IDC(startLineNum=startLineNum,det=det)
def updateProgress(self):
self.progressDialog.setValue(self.progressDialog.value()+1)
def APS_15IDC(self,startLineNum=0, det='CCD'):
"""
Function to read a complete spec File collected at APS 15IDC
"""
self.progressDialog=QProgressDialog('Reading scans form SPEC File:','Abort',0,100)
self.progressDialog.setWindowModality(Qt.WindowModal)
self.progressDialog.setWindowTitle('Wait')
self.progressDialog.setAutoClose(True)
self.progressDialog.setAutoReset(True)
self.progressDialog.setMinimum(1)
self.Data['YCol']='Apex2'
self.Data['NCol']='Monc'
fid=open(self.specfile)
fdata=fid.readlines()
self.SpecFileFull=fdata
fid.close()
if fdata[0][:2]!='#F':
self.Data['NumOfScans']=0
self.Data['Message']='The file is not a valid specfile!!'
print 'Error:: The file is not a valid specfile!!'
else:
startScanLineNums=[i for i in range(startLineNum,len(fdata)) if fdata[i][:2]=='#S']
self.progressDialog.setMaximum(len(startScanLineNums))
self.progressDialog.show()
self.endLineNum=startScanLineNums[-1]
if startLineNum>0:
startScanLineNums=sorted(startScanLineNums)
numOfScans=len(startScanLineNums)
scanLines=[fdata[i] for i in startScanLineNums]
if startLineNum==0:
tmp=0
self.Data['NumOfScans']=0#numOfScans
self.Data['ScanLines']=[]#scanLines
self.Data['StartScanLineNums']=[]#startScanLineNums
else:
tmp=self.Data['NumOfScans']-1
self.Data['NumOfScans']=self.Data['NumOfScans']-1#+numOfScans
self.Data['ScanLines']=self.Data['ScanLines'][:-1]#+scanLines[1:]
self.Data['StartScanLineNums']=self.Data['StartScanLineNums'][:-1]#+startScanLineNums
for i in range(numOfScans):
start=startScanLineNums[i]+1
line=fdata[start]
i=i+tmp
self.Data[i]={}
self.Par[i]={}
if fdata[start-1].split()[2]=='getandsave_mca' or fdata[start-1].split()[2]=='MCAscanpt':
self.Par[i]['Mca']=1
self.Data[i]['ScanVar']='Empty'
else:
self.Par[i]['Mca']=0
tmpdata=[]
self.Par[i]['CCD']=0
while line[:2]!='\n' and line[:2]!='#C':
if line[:2]=='#P':
parName=line[3:].split()
start=start+1
parValue=map(eval,fdata[start][1:].split())
for j in range(len(parName)):
self.Par[i][parName[j]]=parValue[j]
if line[:2]=='#W':
tmppar=line[2:].split()
self.Par[i]['Wavelength']=eval(tmppar[1])
if line[:3]=='#G0':
self.Par[i]['g_l1']=float(line[4:].split()[5])
self.Par[i]['g_l2']=float(line[4:].split()[6])
self.Par[i]['g_l3']=float(line[4:].split()[7])
if line[:2]=='#A':
tmppar=line[2:].split()
self.Par[i]['Absorber']=eval(tmppar[1])
if line[:2]=='#Q':
tmppar=line[2:].split()
self.Par[i]['Q']=map(eval, tmppar)
if line[:3]=='#B0':
tmppar=line[3:].split()
self.Par[i]['CCD']=1
if line[:3]=='#B2':
tmppar=map(eval, line[3:].split())
self.Par[i]['DBPos']=tmppar[:2]
self.Par[i]['S2D_Dist']=tmppar[2]
self.Par[i]['S7D_Dist']=tmppar[3]
if line[:2]=='#L':
scanVar=line[3:-1].split()
self.Data[i]['ScanVar']=scanVar
if line[0]!='#':
tmpdata.append(map(eval, line.split( )))
start=start+1
try:
line=fdata[start]
except:
break
for j in range(len(scanVar)):
try:
self.Data[i][scanVar[j]]=np.array(tmpdata)[:,j]
except:
self.Data[i][scanVar[j]]=None
if len(self.Par[i])==1:
self.Par[i]['Message']='No parameters!!'
self.progressDialog.setLabelText('Reading scans form SPEC File: '+str(i+1))
self.updateProgress()
self.Data['NumOfScans']=i
self.Data['ScanLines']=self.Data['ScanLines']+[scanLines[i-tmp]]
self.Data['StartScanLineNums']=self.Data['StartScanLineNums']+[startScanLineNums[i-tmp]]
self.endLineNum=startScanLineNums[i-tmp]
if self.progressDialog.wasCanceled()==True:
break
self.progressDialog.hide()
def APS_9IDC(self,startLineNum=0, det='CCD'):
"""
Function to read a complete spec File collected at APS 15IDC
"""
self.progressDialog=QProgressDialog('Reading scans form SPEC File:','Abort',0,100)
self.progressDialog.setWindowModality(Qt.WindowModal)
self.progressDialog.setWindowTitle('Wait')
self.progressDialog.setAutoClose(True)
self.progressDialog.setAutoReset(True)
self.progressDialog.setMinimum(1)
self.Data['YCol']='Bicron1'
self.Data['NCol']='i2'
fid=open(self.specfile)
fdata=fid.readlines()
self.SpecFileFull=fdata
fid.close()
if fdata[0][:2]!='#F':
self.Data['NumOfScans']=0
self.Data['Message']='The file is not a valid specfile!!'
print 'Error:: The file is not a valid specfile!!'
else:
startScanLineNums=[i for i in range(startLineNum, len(fdata)) if fdata[i][:2]=='#S']
self.progressDialog.setMaximum(len(startScanLineNums))
self.progressDialog.show()
self.endLineNum=startScanLineNums[-1]
if startLineNum>0:
startScanLineNums=sorted(startScanLineNums)
self.Data['StartScanLineNums']=startScanLineNums
numOfScans=len(self.Data['StartScanLineNums'])
scanLines=[fdata[i] for i in startScanLineNums]
if startLineNum==0:
tmp=0
self.Data['NumOfScans']=0#numOfScans
self.Data['ScanLines']=[]#scanLines
self.Data['StartScanLineNums']=[]#startScanLineNums
self.Par['ParName']=[]
for i in range(startScanLineNums[0]):
line=fdata[i].split()
if fdata[i][:2]=='#O':
self.Par['ParName']=self.Par['ParName']+line[1:]
else:
tmp=self.Data['NumOfScans']
self.Data['NumOfScans']=self.Data['NumOfScans']-1#+numOfScans
self.Data['ScanLines']=self.Data['ScanLines'][:-1]#+scanLines[1:]
self.Data['StartScanLineNums']=self.Data['StartScanLineNums'][:-1]#+startScanLineNums
for i in range(numOfScans):
start=startScanLineNums[i]+1
line=fdata[start]
i=i+tmp
self.Data[i]={}
self.Par[i]={}
if fdata[start-1].split()[2]=='getandsave_mca' or fdata[start-1].split()[2]=='MCAscanpt':
self.Par[i]['Mca']=1
self.Data[i]['ScanVar']='Empty'
else:
self.Par[i]['Mca']=0
self.Par[i]['CCD']=0
tmpdata=[]
pstart=0
while line[:2]!='\n' and line[:2]!='#C':
if line[:2]=='#P':
parValue=map(eval,fdata[start].split()[1:])
for j in range(len(parValue)):
self.Par[i][self.Par['ParName'][pstart]]=parValue[j]
pstart=pstart+1
if line[:2]=='#Q':
tmppar=line[2:].split()
self.Par[i]['Q']=map(eval, tmppar)
if line[:2]=='#L':
scanVar=line[3:-1].split()
self.Data[i]['ScanVar']=scanVar
if line[0]!='#':
tmpdata.append(map(eval, line.split( )))
start=start+1
line=fdata[start]
for j in range(len(scanVar)):
try:
self.Data[i][scanVar[j]]=np.array(tmpdata)[:,j]
except:
self.Data[i][scanVar[j]]=None
if len(self.Par[i])==1:
self.Par[i]['Message']='No parameters!!'
self.progressDialog.setLabelText('Reading scans form SPEC File: '+str(i+1))
self.updateProgress()
self.Data['NumOfScans']=i
self.Data['ScanLines']=self.Data['ScanLines']+[scanLines[i-tmp]]
self.Data['StartScanLineNums']=self.Data['StartScanLineNums']+[startScanLineNums[i-tmp]]
self.endLineNum=startScanLineNums[i-tmp]
if self.progressDialog.wasCanceled()==True:
break
self.progressDialog.hide()
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = "http://osm.linfiniti.com/buildings-shp"
self.roads_url = "http://osm.linfiniti.com/roads-shp"
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr("InaSAFE OpenStreetMap Downloader")
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if proxy is not None:
self.network_manager.setProxy(proxy)
self.restore_state()
self.update_extent()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent will be used to determine the area for which '
'you want data to be retrieved. You can adjust it manually using '
'the bounding box options below.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self):
""" Update extent value in GUI based from value in map."""
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
def validate_extent(self):
"""Validate the bounding box before user click OK to download.
:return: True if the bounding box is valid, otherwise False
:rtype: bool
"""
min_latitude = float(str(self.min_latitude.text()))
max_latitude = float(str(self.max_latitude.text()))
min_longitude = float(str(self.min_longitude.text()))
max_longitude = float(str(self.max_longitude.text()))
# min_latitude < max_latitude
if min_latitude >= max_latitude:
return False
# min_longitude < max_longitude
if min_longitude >= max_longitude:
return False
# -90 <= latitude <= 90
if min_latitude < -90 or min_latitude > 90:
return False
if max_latitude < -90 or max_latitude > 90:
return False
# -180 <= longitude <= 180
if min_longitude < -180 or min_longitude > 180:
return False
if max_longitude < -180 or max_longitude > 180:
return False
return True
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
""" Show a dialog to choose directory """
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select download directory")))
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Validate extent
valid_flag = self.validate_extent()
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(self, error_dialog_title, message)
return
# Get all the feature types
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads']
elif index == 1:
feature_types = ['buildings']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
self.download(feature_type)
self.load_shapefile(feature_type)
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(self, error_dialog_title, str(exception))
self.progress_dialog.cancel()
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = str(self.output_directory.text())
if os.path.exists(path):
return
title = self.tr("Directory %s not exist") % path
question = self.tr(
"Directory %s not exist. Do you want to create it?") % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
QMessageBox.warning(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type):
"""Download shapefiles from Linfiniti server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError, CanceledImportDialogError
"""
# preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = (
'{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(
min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude
)
output_prefix = self.filename_prefix.text()
if feature_type == 'buildings':
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.buildings_url, box=box)
else:
url = "{url}?bbox={box}&qgis_version=2".format(
url=self.roads_url, box=box)
if output_prefix is not None:
url += '&output_prefix=%s' % output_prefix
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path)
self.extract_zip(path, str(self.output_directory.text()))
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
self.progress_dialog.setMaximum(100)
self.progress_dialog.setValue(0)
label_text = self.tr("Downloading shapefile")
self.progress_dialog.setLabelText(label_text)
# Download Process
downloader = FileDownloader(
self.network_manager, url, output_path, self.progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
raise DownloadError(error_message)
@staticmethod
def extract_zip(path, output_dir):
"""Extract all content of a .zip file from path to output_dir.
:param path: The path of the .zip file
:type path: str
:param output_dir: Output directory where the shp will be written to.
:type output_dir: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
# extract all files...
handle = open(path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
output_path = os.path.join(output_dir, name)
output_file = open(output_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when buildings.shp not exist
"""
output_prefix = self.filename_prefix.text()
path = str(self.output_directory.text())
path = os.path.join(path, '%s%s.shp' % (output_prefix, feature_type))
if not os.path.exists(path):
message = self.tr(
"%s don't exist. The server doesn't have any data.")
raise ImportDialogError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('InaSAFE OpenStreetMap Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up things for context help
self.help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
# Allow toggling the help button
self.help_button.setCheckable(True)
self.help_button.toggled.connect(self.help_toggled)
self.main_stacked_widget.setCurrentIndex(1)
# Disable boundaries group box until boundary checkbox is ticked
self.boundary_group.setEnabled(False)
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
self.restore_state()
# Setup the rectangle map tool
self.canvas = iface.mapCanvas()
self.rectangle_map_tool = \
RectangleMapTool(self.canvas)
self.rectangle_map_tool.rectangle_created.connect(
self.update_extent_from_rectangle)
self.capture_button.clicked.connect(self.drag_rectangle_on_map_canvas)
# Setup pan tool
self.pan_tool = QgsMapToolPan(self.canvas)
self.canvas.setMapTool(self.pan_tool)
# Setup helper for admin_level
json_file_path = resources_path('osm', 'admin_level_per_country.json')
if os.path.isfile(json_file_path):
self.countries = json.load(open(json_file_path))
self.bbox_countries = None
self.populate_countries()
# connect
self.country_comboBox.currentIndexChanged.connect(
self.update_helper_political_level)
self.admin_level_comboBox.currentIndexChanged.connect(
self.update_helper_political_level)
self.update_extent_from_map_canvas()
def update_helper_political_level(self):
"""To update the helper about the country and the admin_level."""
current_country = self.country_comboBox.currentText()
index = self.admin_level_comboBox.currentIndex()
current_level = self.admin_level_comboBox.itemData(index)
content = None
try:
content = \
self.countries[current_country]['levels'][str(current_level)]
if content == 'N/A' or content == 'fixme' or content == '':
raise KeyError
except KeyError:
content = self.tr('undefined')
finally:
text = self.tr('which represents %s in') % content
self.boundary_helper.setText(text)
def populate_countries(self):
"""Populate the combobox about countries and levels."""
for i in range(1, 12):
self.admin_level_comboBox.addItem(self.tr('Level %s') % i, i)
# Set current index to admin_level 8, the most common one
self.admin_level_comboBox.setCurrentIndex(7)
list_countries = [
self.tr(country) for country in self.countries.keys()
]
list_countries.sort()
for country in list_countries:
self.country_comboBox.addItem(country)
self.bbox_countries = {}
for country in list_countries:
multipolygons = self.countries[country]['bbox']
self.bbox_countries[country] = []
for coords in multipolygons:
bbox = QgsRectangle(coords[0], coords[3], coords[2], coords[1])
self.bbox_countries[country].append(bbox)
self.update_helper_political_level()
@pyqtSlot()
@pyqtSignature('bool') # prevents actions being handled twice
def help_toggled(self, flag):
"""Show or hide the help tab in the stacked widget.
.. versionadded: 3.2
:param flag: Flag indicating whether help should be shown or hidden.
:type flag: bool
"""
if flag:
self.help_button.setText(self.tr('Hide Help'))
self.show_help()
else:
self.help_button.setText(self.tr('Show Help'))
self.hide_help()
def hide_help(self):
"""Hide the usage info from the user.
.. versionadded:: 3.2
"""
self.main_stacked_widget.setCurrentIndex(1)
def show_help(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
self.main_stacked_widget.setCurrentIndex(0)
header = html_header()
footer = html_footer()
string = header
message = osm_downloader_help()
string += message.to_html()
string += footer
self.help_web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def update_extent(self, extent):
"""Update extent value in GUI based from an extent.
:param extent: A list in the form [xmin, ymin, xmax, ymax] where all
coordinates provided are in Geographic / EPSG:4326.
:type extent: list
"""
self.x_minimum.setValue(extent[0])
self.y_minimum.setValue(extent[1])
self.x_maximum.setValue(extent[2])
self.y_maximum.setValue(extent[3])
# Updating the country if possible.
rectangle = QgsRectangle(extent[0], extent[1], extent[2], extent[3])
center = rectangle.center()
for country in self.bbox_countries:
for polygon in self.bbox_countries[country]:
if polygon.contains(center):
index = self.country_comboBox.findText(country)
self.country_comboBox.setCurrentIndex(index)
break
else:
# Continue if the inner loop wasn't broken.
continue
# Inner loop was broken, break the outer.
break
else:
self.country_comboBox.setCurrentIndex(0)
def update_extent_from_map_canvas(self):
"""Update extent value in GUI based from value in map.
.. note:: Delegates to update_extent()
"""
self.bounding_box_group.setTitle(
self.tr('Bounding box from the map canvas'))
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.update_extent(extent)
def update_extent_from_rectangle(self):
"""Update extent value in GUI based from the QgsMapTool rectangle.
.. note:: Delegates to update_extent()
"""
self.show()
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.canvas.setMapTool(self.pan_tool)
rectangle = self.rectangle_map_tool.rectangle()
if rectangle:
self.bounding_box_group.setTitle(
self.tr('Bounding box from rectangle'))
extent = rectangle_geo_array(rectangle, self.iface.mapCanvas())
self.update_extent(extent)
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory."""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(
QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def drag_rectangle_on_map_canvas(self):
"""Hide the dialog and allow the user to draw a rectangle."""
self.hide()
self.rectangle_map_tool.reset()
self.canvas.unsetMapTool(self.pan_tool)
self.canvas.setMapTool(self.rectangle_map_tool)
def get_checked_features(self):
"""Create a tab with all checked features.
:return A list with all features which are checked in the UI.
:rtype list
"""
feature_types = []
if self.roads_flag.isChecked():
feature_types.append('roads')
if self.buildings_flag.isChecked():
feature_types.append('buildings')
if self.building_points_flag.isChecked():
feature_types.append('building-points')
if self.flood_prone_flag.isChecked():
feature_types.append('flood-prone')
if self.evacuation_centers_flag.isChecked():
feature_types.append('evacuation-centers')
if self.boundary_flag.isChecked():
level = self.admin_level_comboBox.currentIndex() + 1
feature_types.append('boundary-%s' % level)
return feature_types
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Lock the bounding_box_group
self.bounding_box_group.setDisabled(True)
# Get the extent
y_minimum = self.y_minimum.value()
y_maximum = self.y_maximum.value()
x_minimum = self.x_minimum.value()
x_maximum = self.x_maximum.value()
extent = [x_minimum, y_minimum, x_maximum, y_maximum]
# Validate extent
valid_flag = validate_geo_array(extent)
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the bounding_box_group
self.bounding_box_group.setEnabled(True)
return
# Validate features
feature_types = self.get_checked_features()
if len(feature_types) < 1:
message = self.tr('No feature selected. '
'Please make sure you have checked one feature.')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the bounding_box_group
self.bounding_box_group.setEnabled(True)
return
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_flag.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, feature_type, overwrite)
# noinspection PyTypeChecker
download(feature_type, output_base_file_path, extent,
self.progress_dialog)
try:
self.load_shapefile(feature_type, output_base_file_path)
except FileMissingError as exception:
display_warning_message_box(self, error_dialog_title,
exception.message)
self.done(QDialog.Accepted)
self.rectangle_map_tool.reset()
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title,
exception.message)
self.progress_dialog.cancel()
finally:
# Unlock the bounding_box_group
self.bounding_box_group.setEnabled(True)
def get_output_base_path(self, output_directory, output_prefix,
feature_type, overwrite):
"""Get a full base name path to save the shapefile.
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
path = os.path.join(output_directory,
'%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix('%s.shp' % path,
separator)
if suffix:
path = os.path.join(
output_directory, '%s%s%s%s' %
(output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return OsmDownloaderDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(self, title, question,
QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
layer = self.iface.addVectorLayer(path, feature_type, 'ogr')
# Check if it's a building layer and if it's QGIS 2.14 about the 2.5D
if qgis_version() >= 21400 and feature_type == 'buildings':
layer_scope = QgsExpressionContextUtils.layerScope(layer)
if not layer_scope.variable('qgis_25d_height'):
QgsExpressionContextUtils.setLayerVariable(
layer, 'qgis_25d_height', 0.0002)
if not layer_scope.variable('qgis_25d_angle'):
QgsExpressionContextUtils.setLayerVariable(
layer, 'qgis_25d_angle', 70)
canvas_srid = self.canvas.mapSettings().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.iface, self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers'))
def reject(self):
"""Redefinition of the method to remove the rectangle selection tool.
It will call the super method.
"""
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.rectangle_map_tool.reset()
super(OsmDownloaderDialog, self).reject()
class WCPSClientDialog(QtGui.QDialog, Ui_WCPSClient):
def __init__(self, iface):
"""Constructor."""
QtGui.QDialog.__init__(self)
# Set up the user interface from Designer.
# After setupUI you can access any designer object by doing
# self.<objectname>, and you can use autoconnect slots - see
# http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html
# #widgets-and-dialogs-with-auto-connect
self.setupUi(self)
self.iface = iface
if len(config.srv_list['servers']) > 0:
self.btnEdit_Serv.setEnabled(True)
self.btnDelete_Serv.setEnabled(True)
self.updateServerListing()
self.myWCPS = WCPSUtil()
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(True) # False # was set originally
title = self.tr("WCPS Client 1.0")
self.progress_dialog.setWindowTitle(title)
self.tabWidget_WCPSClient.setCurrentIndex(0)
#---------------
# add a new server to the list
def newServer(self):
global config
#print 'btnNew: I am adding a New ServerName/URL'
flags = Qt.WindowTitleHint | Qt.WindowSystemMenuHint | Qt.WindowMinimizeButtonHint | Qt.WindowMaximizeButtonHint
dlgNew = qgsnewhttpconnectionbase(self, flags, toEdit=False, choice='')
dlgNew.show()
self.btnConnectServer_Serv.setFocus(True)
##TODO -- sort the srv_list
#---------------
# read the selected server/url params
def get_serv_url(self):
global serv
sel_serv = self.cmbConnections_Serv.currentText()
idx = serv.index(sel_serv)
sel_url = config.srv_list['servers'][idx][1]
return sel_serv, sel_url
#---------------
# check if the url exist and if we get a respond to a simple OWS request
@mouse_busy
def connectServer(self):
global config
global serv
selected_serv, selected_url = self.get_serv_url()
print 'You choose: ', selected_serv, "URL:", selected_url
msg = "Your choice: " + selected_serv.encode() + "\n"
msg = msg + "URL: " + selected_url.encode() + "\n"
self.textBrowser_Serv.setText(msg)
if not self.tab_PC.isEnabled():
self.tab_PC.setEnabled(True)
QApplication.changeOverrideCursor(Qt.ArrowCursor)
# modify a server entry
def editServer(self):
global config
#print "btnEdit: here we are editing... "
flags = Qt.WindowTitleHint | Qt.WindowSystemMenuHint | Qt.WindowMinimizeButtonHint | Qt.WindowMaximizeButtonHint
idx = self.cmbConnections_Serv.currentIndex()
if idx < len(config.srv_list['servers']):
select_serv = config.srv_list['servers'][idx]
print "Selection: ", idx, " -- ", select_serv, " -- Check: ", serv[
idx]
dlgEdit = qgsnewhttpconnectionbase(self,
flags,
toEdit=True,
choice=idx)
dlgEdit.txt_NewSrvName.setText(select_serv[0])
dlgEdit.txt_NewSrvUrl.setText(select_serv[1])
dlgEdit.show()
self.btnConnectServer_Serv.setFocus(True)
#---------------
#---------------
# dele a server entry
def deleteServer(self):
global config
#print "btnDelete: here we are deleting...."
idx = self.cmbConnections_Serv.currentIndex()
if idx < len(config.srv_list['servers']):
config.srv_list['servers'].pop(idx)
self.write_srv_list()
self.updateServerListing()
self.btnConnectServer_Serv.setFocus(True)
#---------------
#---------------
# update the server-listing shown in the selectionBar
def updateServerListing(self):
global serv
global config
#print "btnUpdateServerListing: here we are updating the ServerList...."
serv = []
config.srv_list = config.read_srv_list()
for ii in range(len(config.srv_list['servers'])):
serv.append(config.srv_list['servers'][ii][0][:])
self.cmbConnections_Serv.clear()
self.cmbConnections_Serv.addItems(serv)
#---------------
#---------------
# write the sever names/urls to a file
@mouse_busy
def write_srv_list(self):
#print "btnwriteServerListing: here we are writing the ServerList...."
plugin_dir = os.path.dirname(os.path.realpath(__file__))
outsrvlst = os.path.join(plugin_dir, 'config_srvlist.pkl')
fo = open(outsrvlst, 'wb')
pickle.dump(config.srv_list, fo, 0)
fo.close()
#---------------
#---------------
# get the path where the downloaded datasets shall be stored
@mouse_busy
def get_outputLoc(self):
global req_outputLoc
start_dir = os.getenv("HOME")
req_outputLoc = QFileDialog.getExistingDirectory(
self, "Select Output Path", start_dir)
if len(req_outputLoc) > 0:
if not req_outputLoc.endswith(os.sep):
req_outputLoc = req_outputLoc + os.sep
self.lineEdit_path.setText(req_outputLoc)
#print req_outputLoc
## ====== End of Server section ======
@mouse_busy
def exeProcessCoverage(self):
global req_outputLoc
selected_serv, selected_url = self.get_serv_url()
query = self.plainTextEdit_PC.toPlainText()
if query is None:
msg = "Please enter a query"
warning_msg(msg)
return
linePath = self.lineEdit_path.text()
if len(linePath) == 0:
if not "req_outputLoc" in globals():
self.tabWidget_WCPSClient.setCurrentIndex(1)
self.get_outputLoc()
elif len(req_outputLoc) == 0:
self.tabWidget_WCPSClient.setCurrentIndex(1)
self.get_outputLoc()
else:
req_outputLoc = linePath
#display the directory no matter what
self.lineEdit_path.setText(req_outputLoc)
input_param = {
'query': query,
'outputDir': req_outputLoc,
'serv_url': selected_url
}
#self.add_to_map(self.myWCPS.ProcessCoverage(input_param))
process_output = self.myWCPS.ProcessCoverage(input_param)
status = -1
outLoc = ''
mimetype = ''
dialogMessage = ''
if (process_output.has_key('status')):
status = process_output['status']
if (process_output.has_key('outfile')):
outLoc = process_output['outfile']
if (process_output.has_key('mimetype')):
mimetype = process_output['mimetype']
if status == 200:
datatype = mimetype.split('/')
if datatype[0] == "image":
self.add_to_map(self.myWCPS.ProcessCoverage(input_param))
elif datatype[0] == "application":
self.add_to_map(self.myWCPS.ProcessCoverage(input_param))
else:
showData = open(outLoc, 'r')
dialogMessage = showData.read()
myDisplay_txt = display_txt(self)
myDisplay_txt.textBrowser_Disp.setText(dialogMessage)
myDisplay_txt.show()
else:
myDisplay_txt = display_txt(self)
myDisplay_txt.textBrowser_Disp.setText(process_output['message'])
myDisplay_txt.show()
## ====== Add data to Map Canvas ======
# read the the downloaded datasets, register them and show them in the QGis MapCanvas
def add_to_map(self, req_params):
self.canvas = self.iface.mapCanvas()
fileID = req_params['outfile']
disp_image = glob(fileID)
# check if there is a loadable coverage availabel (and not eg. an multipart/related gml) or an error occurred
if len(disp_image) > 0:
imgInfo = QFileInfo(disp_image[-1])
img_baseName = imgInfo.baseName()
img_layer = QgsRasterLayer(disp_image[-1], img_baseName.encode())
if not img_layer.isValid():
warning_msg("Layer failed to load!")
else:
msg = "Could not load file"
warning_msg(msg)
QgsMapLayerRegistry.instance().addMapLayer(img_layer)
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.buildings_url = 'http://osm.linfiniti.com/buildings-shp'
self.building_points_url = \
'http://osm.linfiniti.com/building-points-shp'
self.roads_url = 'http://osm.linfiniti.com/roads-shp'
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('InaSAFE OpenStreetMap Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up context help
help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
help_button.clicked.connect(self.show_help)
self.show_info()
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
# Set Proxy in webpage
proxy = get_proxy()
self.network_manager = QNetworkAccessManager(self)
if proxy is not None:
self.network_manager.setProxy(proxy)
self.restore_state()
# Setup the rectangle map tool
self.canvas = iface.mapCanvas()
self.rectangle_map_tool = \
RectangleMapTool(self.canvas)
self.rectangle_map_tool.rectangle_created.connect(
self.update_extent_from_rectangle)
self.button_extent_rectangle.clicked.connect(
self.drag_rectangle_on_map_canvas)
# Setup pan tool
self.pan_tool = QgsMapToolPan(self.canvas)
self.canvas.setMapTool(self.pan_tool)
self.update_extent_from_map_canvas()
def show_info(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent, when opening this window, will be used to '
'determine the area for which you want data to be retrieved.'
'You can interactively select the area by using the '
'\'select on map\' button - which will temporarily hide this '
'window and allow you to drag a rectangle on the map. After you '
'have finished dragging the rectangle, this window will '
'reappear.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def show_help(self):
"""Load the help text for the dialog."""
show_context_help(self.help_context)
def update_extent(self, extent):
"""Update extent value in GUI based from an extent.
:param extent: A list in the form [xmin, ymin, xmax, ymax] where all
coordinates provided are in Geographic / EPSG:4326.
:type extent: list
"""
self.min_longitude.setText(str(extent[0]))
self.min_latitude.setText(str(extent[1]))
self.max_longitude.setText(str(extent[2]))
self.max_latitude.setText(str(extent[3]))
def update_extent_from_map_canvas(self):
"""Update extent value in GUI based from value in map.
.. note:: Delegates to update_extent()
"""
self.groupBox.setTitle(self.tr('Bounding box from the map canvas'))
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.update_extent(extent)
def update_extent_from_rectangle(self):
"""Update extent value in GUI based from the QgsMapTool rectangle.
.. note:: Delegates to update_extent()
"""
self.show()
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.canvas.setMapTool(self.pan_tool)
rectangle = self.rectangle_map_tool.rectangle()
if rectangle:
self.groupBox.setTitle(self.tr('Bounding box from rectangle'))
extent = rectangle_geo_array(rectangle, self.iface.mapCanvas())
self.update_extent(extent)
def validate_extent(self):
"""Validate the bounding box before user click OK to download.
:return: True if the bounding box is valid, otherwise False
:rtype: bool
"""
min_latitude = float(str(self.min_latitude.text()))
max_latitude = float(str(self.max_latitude.text()))
min_longitude = float(str(self.min_longitude.text()))
max_longitude = float(str(self.max_longitude.text()))
# min_latitude < max_latitude
if min_latitude >= max_latitude:
return False
# min_longitude < max_longitude
if min_longitude >= max_longitude:
return False
# -90 <= latitude <= 90
if min_latitude < -90 or min_latitude > 90:
return False
if max_latitude < -90 or max_latitude > 90:
return False
# -180 <= longitude <= 180
if min_longitude < -180 or min_longitude > 180:
return False
if max_longitude < -180 or max_longitude > 180:
return False
return True
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory."""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def drag_rectangle_on_map_canvas(self):
"""Hide the dialog and allow the user to draw a rectangle."""
self.hide()
self.rectangle_map_tool.reset()
self.canvas.unsetMapTool(self.pan_tool)
self.canvas.setMapTool(self.rectangle_map_tool)
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Lock the groupbox
self.groupBox.setDisabled(True)
# Validate extent
valid_flag = self.validate_extent()
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the groupbox
self.groupBox.setEnabled(True)
return
# Get all the feature types
index = self.feature_type.currentIndex()
if index == 0:
feature_types = ['buildings', 'roads', 'building-points']
elif index == 1:
feature_types = ['buildings']
elif index == 2:
feature_types = ['building-points']
else:
feature_types = ['roads']
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_checkBox.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, feature_type, overwrite)
self.download(feature_type, output_base_file_path)
try:
self.load_shapefile(feature_type, output_base_file_path)
except FileMissingError as exception:
display_warning_message_box(
self,
error_dialog_title,
exception.message)
self.done(QDialog.Accepted)
self.rectangle_map_tool.reset()
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, error_dialog_title, exception.message)
self.progress_dialog.cancel()
finally:
# Unlock the groupbox
self.groupBox.setEnabled(True)
def get_output_base_path(
self,
output_directory,
output_prefix,
feature_type,
overwrite):
"""Get a full base name path to save the shapefile.
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
path = os.path.join(
output_directory, '%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix(
'%s.shp' % path, separator)
if suffix:
path = os.path.join(output_directory, '%s%s%s%s' % (
output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return OsmDownloaderDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def download(self, feature_type, output_base_path):
"""Download shapefiles from Kartoza server.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param output_base_path: The base path of the shape file.
:type output_base_path: str
:raises: ImportDialogError, CanceledImportDialogError
"""
# preparing necessary data
min_longitude = str(self.min_longitude.text())
min_latitude = str(self.min_latitude.text())
max_longitude = str(self.max_longitude.text())
max_latitude = str(self.max_latitude.text())
box = (
'{min_longitude},{min_latitude},{max_longitude},'
'{max_latitude}').format(
min_longitude=min_longitude,
min_latitude=min_latitude,
max_longitude=max_longitude,
max_latitude=max_latitude
)
if feature_type == 'buildings':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.buildings_url, box=box)
elif feature_type == 'building-points':
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.building_points_url, box=box)
else:
url = '{url}?bbox={box}&qgis_version=2'.format(
url=self.roads_url, box=box)
if 'LANG' in os.environ:
env_lang = os.environ['LANG']
url += '&lang=%s' % env_lang
path = tempfile.mktemp('.shp.zip')
# download and extract it
self.fetch_zip(url, path, feature_type)
self.extract_zip(path, output_base_path)
self.progress_dialog.done(QDialog.Accepted)
def fetch_zip(self, url, output_path, feature_type):
"""Download zip containing shp file and write to output_path.
:param url: URL of the zip bundle.
:type url: str
:param output_path: Path of output file,
:type output_path: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:raises: ImportDialogError - when network error occurred
"""
LOGGER.debug('Downloading file from URL: %s' % url)
LOGGER.debug('Downloading to: %s' % output_path)
self.progress_dialog.show()
# Infinite progress bar when the server is fetching data.
# The progress bar will be updated with the file size later.
self.progress_dialog.setMaximum(0)
self.progress_dialog.setMinimum(0)
self.progress_dialog.setValue(0)
# Get a pretty label from feature_type, but not translatable
label_feature_type = feature_type.replace('-', ' ')
label_text = self.tr('Fetching %s' % label_feature_type)
self.progress_dialog.setLabelText(label_text)
# Download Process
downloader = FileDownloader(
self.network_manager, url, output_path, self.progress_dialog)
try:
result = downloader.download()
except IOError as ex:
raise IOError(ex)
if result[0] is not True:
_, error_message = result
if result[0] == QNetworkReply.OperationCanceledError:
raise CanceledImportDialogError(error_message)
else:
raise DownloadError(error_message)
@staticmethod
def extract_zip(zip_path, destination_base_path):
"""Extract different extensions to the destination base path.
Example : test.zip contains a.shp, a.dbf, a.prj
and destination_base_path = '/tmp/CT-buildings
Expected result :
- /tmp/CT-buildings.shp
- /tmp/CT-buildings.dbf
- /tmp/CT-buildings.prj
If two files in the zip with the same extension, only one will be
copied.
:param zip_path: The path of the .zip file
:type zip_path: str
:param destination_base_path: The destination base path where the shp
will be written to.
:type destination_base_path: str
:raises: IOError - when not able to open path or output_dir does not
exist.
"""
import zipfile
handle = open(zip_path, 'rb')
zip_file = zipfile.ZipFile(handle)
for name in zip_file.namelist():
extension = os.path.splitext(name)[1]
output_final_path = u'%s%s' % (destination_base_path, extension)
output_file = open(output_final_path, 'wb')
output_file.write(zip_file.read(name))
output_file.close()
handle.close()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers')
)
def reject(self):
"""Redefinition of the reject() method
to remove the rectangle selection tool.
It will call the super method.
"""
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.rectangle_map_tool.reset()
super(OsmDownloaderDialog, self).reject()
class PetaBencanaDialog(QDialog, FORM_CLASS):
"""Downloader for PetaBencana data.
.. versionadded: 3.3
"""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
.. versionadded: 3.3
:param parent: Optional widget to use as parent.
:type parent: QWidget
:param iface: An instance of QGisInterface.
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
title = self.tr('PetaBencana Downloader')
self.setWindowTitle(title)
self.iface = iface
self.source = None
self.radio_button_group = QButtonGroup()
self.radio_button_group.addButton(self.radio_button_production)
self.radio_button_group.addButton(self.radio_button_development)
self.radio_button_group.setExclusive(True)
self.radio_button_production.setChecked(True)
self.populate_combo_box()
developer_mode = setting('developer_mode', False, bool)
if not developer_mode:
self.radio_button_widget.hide()
self.source_label.hide()
self.output_group.adjustSize()
# signals
self.radio_button_production.clicked.connect(self.populate_combo_box)
self.radio_button_development.clicked.connect(self.populate_combo_box)
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
self.progress_dialog.setWindowTitle(title)
# Set up things for context help
self.help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
# Allow toggling the help button
self.help_button.setCheckable(True)
self.help_button.toggled.connect(self.help_toggled)
self.main_stacked_widget.setCurrentIndex(1)
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
self.time_stamp = None
self.restore_state()
@pyqtSlot()
@pyqtSignature('bool') # prevents actions being handled twice
def help_toggled(self, flag):
"""Show or hide the help tab in the stacked widget.
.. versionadded: 3.3
:param flag: Flag indicating whether help should be shown or hidden.
:type flag: bool
"""
if flag:
self.help_button.setText(self.tr('Hide Help'))
self.show_help()
else:
self.help_button.setText(self.tr('Show Help'))
self.hide_help()
def hide_help(self):
"""Hide the usage info from the user.
.. versionadded:: 3.3
"""
self.main_stacked_widget.setCurrentIndex(1)
def show_help(self):
"""Show usage info to the user.
.. versionadded: 3.3
"""
# Read the header and footer html snippets
self.main_stacked_widget.setCurrentIndex(0)
header = html_header()
footer = html_footer()
string = header
message = peta_bencana_help()
string += message.to_html()
string += footer
self.help_web_view.setHtml(string)
def restore_state(self):
"""Read last state of GUI from configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
"""Store current state of GUI to configuration file.
.. versionadded: 3.3
"""
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory.
.. versionadded: 3.3
"""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def accept(self):
"""Do PetaBencana download and display it in QGIS.
.. versionadded: 3.3
"""
self.save_state()
try:
self.require_directory()
except CanceledImportDialogError:
return
QtGui.qApp.setOverrideCursor(QtGui.QCursor(QtCore.Qt.WaitCursor))
source = self.define_url()
# save the file as json first
name = 'jakarta_flood.json'
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_flag.isChecked()
date_stamp_flag = self.include_date_flag.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory,
output_prefix,
date_stamp_flag,
name,
overwrite)
title = self.tr("Can't access API")
try:
self.download(source, output_base_file_path)
# Open downloaded file as QgsMapLayer
layer = QgsVectorLayer(
output_base_file_path, 'flood', 'ogr', False)
except Exception as e:
disable_busy_cursor()
QMessageBox.critical(self, title, str(e))
return
self.time_stamp = time.strftime('%d-%b-%Y %H:%M:%S')
# Now save as shp
name = 'jakarta_flood.shp'
output_base_file_path = self.get_output_base_path(
output_directory,
output_prefix,
date_stamp_flag,
name,
overwrite)
QgsVectorFileWriter.writeAsVectorFormat(
layer, output_base_file_path, 'CP1250', None, 'ESRI Shapefile')
# Get rid of the GeoJSON layer and rather use local shp
del layer
self.copy_style(output_base_file_path)
self.copy_keywords(output_base_file_path)
layer = self.add_flooded_field(output_base_file_path)
# check if the layer has feature or not
if layer.featureCount() <= 0:
city = self.city_combo_box.currentText()
message = self.tr(
'There are no floods data available on {city} '
'at this time.').format(city=city)
display_warning_message_box(
self,
self.tr('No data'),
message)
disable_busy_cursor()
else:
# add the layer to the map
registry = QgsMapLayerRegistry.instance()
registry.addMapLayer(layer)
disable_busy_cursor()
self.done(QDialog.Accepted)
def add_flooded_field(self, shapefile_path):
"""Create the layer from the local shp adding the flooded field.
.. versionadded:: 3.3
Use this method to add a calculated field to a shapefile. The shapefile
should have a field called 'count' containing the number of flood
reports for the field. The field values will be set to 0 if the count
field is < 1, otherwise it will be set to 1.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
:return: A vector layer with the flooded field added.
:rtype: QgsVectorLayer
"""
layer = QgsVectorLayer(
shapefile_path, self.tr('Jakarta Floods'), 'ogr')
# Add a calculated field indicating if a poly is flooded or not
# from PyQt4.QtCore import QVariant
layer.startEditing()
# Add field with integer from 0 to 4 which represents the flood
# class. Its the same as 'state' field except that is being treated
# as a string.
# This is used for cartography
flood_class_field = QgsField('floodclass', QVariant.Int)
layer.dataProvider().addAttributes([flood_class_field])
layer.commitChanges()
layer.startEditing()
flood_class_idx = layer.fieldNameIndex('floodclass')
flood_class_expression = QgsExpression('to_int(state)')
context = QgsExpressionContext()
context.setFields(layer.pendingFields())
flood_class_expression.prepare(context)
# Add field with boolean flag to say if the area is flooded
# This is used by the impact function
flooded_field = QgsField('flooded', QVariant.Int)
layer.dataProvider().addAttributes([flooded_field])
layer.commitChanges()
layer.startEditing()
flooded_idx = layer.fieldNameIndex('flooded')
flood_flag_expression = QgsExpression('state > 0')
flood_flag_expression.prepare(context)
for feature in layer.getFeatures():
context.setFeature(feature)
feature[flood_class_idx] = flood_class_expression.evaluate(context)
feature[flooded_idx] = flood_flag_expression.evaluate(context)
layer.updateFeature(feature)
layer.commitChanges()
return layer
def copy_keywords(self, shapefile_path):
"""Copy keywords from the OSM resource directory to the output path.
.. versionadded: 3.3
In addition to copying the template, tokens within the template will
be replaced with new values for the date token and title token.
:param shapefile_path: Path to the shapefile that will have the flooded
field added.
:type shapefile_path: basestring
"""
source_xml_path = resources_path('petabencana', 'flood-keywords.xml')
output_xml_path = shapefile_path.replace('shp', 'xml')
LOGGER.info('Copying xml to: %s' % output_xml_path)
title_token = '[TITLE]'
new_title = self.tr('Jakarta Floods - %s' % self.time_stamp)
date_token = '[DATE]'
new_date = self.time_stamp
with open(source_xml_path) as source_file, \
open(output_xml_path, 'w') as output_file:
for line in source_file:
line = line.replace(date_token, new_date)
line = line.replace(title_token, new_title)
output_file.write(line)
@staticmethod
def copy_style(shapefile_path):
"""Copy style from the OSM resource directory to the output path.
.. versionadded: 3.3
:param shapefile_path: Path to the shapefile that should get the path
added.
:type shapefile_path: basestring
"""
source_qml_path = resources_path('petabencana', 'flood-style.qml')
output_qml_path = shapefile_path.replace('shp', 'qml')
LOGGER.info('Copying qml to: %s' % output_qml_path)
copy(source_qml_path, output_qml_path)
def get_output_base_path(
self,
output_directory,
output_prefix,
with_date_stamp,
feature_type,
overwrite):
"""Get a full base name path to save the shapefile.
TODO: This is cut & paste from OSM - refactor to have one method
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param with_date_stamp: Whether to add a datestamp in between the
file prefix and the feature_type for the shapefile name.
:type output_prefix: str
:param feature_type: What kind of data will be downloaded. Will be
used for the shapefile name.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
if with_date_stamp and self.time_stamp is not None:
time_stamp = self.time_stamp.replace(' ', '-')
time_stamp = time_stamp.replace(':', '-')
time_stamp += '-'
feature_type = time_stamp + feature_type
path = os.path.join(
output_directory, '%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix(
'%s.shp' % path, separator)
if suffix:
path = os.path.join(output_directory, '%s%s%s%s' % (
output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
TODO: This is cut & paste from OSM - refactor to have one method
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return PetaBencanaDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
TODO: This is cut & paste from OSM - refactor to have one method
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
TODO: This is cut & paste from OSM - refactor to have one method
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapSettings().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.iface,
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers')
)
def reject(self):
"""Redefinition of the method.
It will call the super method.
"""
super(PetaBencanaDialog, self).reject()
def download(self, url, output_path):
"""Download file from API url and write to output path.
:param url: URL of the API.
:type url: str
:param output_path: Path of output file,
:type output_path: str
"""
request_failed_message = self.tr(
"Can't access PetaBencana API: {source}").format(
source=url)
downloader = FileDownloader(url, output_path)
result, message = downloader.download()
if not result:
display_warning_message_box(
self,
self.tr('Download error'),
self.tr(request_failed_message + '\n' + message))
if result == QNetworkReply.OperationCanceledError:
display_warning_message_box(
self,
self.tr('Download error'),
self.tr(message))
# The function below might be usefull for future usage.
# def get_available_area(self):
# """Function to automatically get the available area on API.
# *still cannot get string data from QByteArray*
# """
# available_area = []
# network_manager = QgsNetworkAccessManager.instance()
# api_url = QUrl('https://data.petabencana.id/cities')
# api_request = QNetworkRequest(api_url)
# api_response = network_manager.get(api_request)
# data = api_response.readAll()
# json_response = QScriptEngine().evaluate(data)
# geometries = json_response.property('output').property('geometries')
# iterator = QScriptValueIterator(geometries)
# while iterator.hasNext():
# iterator.next()
# geometry = iterator.value()
# geometry_code = (
# geometry.property('properties').property('code').toString())
# available_area.append(geometry_code)
def populate_combo_box(self):
"""Populate combobox for selecting city."""
if self.radio_button_production.isChecked():
self.source = production_api['url']
available_data = production_api['available_data']
else:
self.source = development_api['url']
available_data = development_api['available_data']
self.city_combo_box.clear()
for index, data in enumerate(available_data):
self.city_combo_box.addItem(data['name'])
self.city_combo_box.setItemData(
index, data['code'], Qt.UserRole)
def define_url(self):
"""Define API url based on which source is selected.
:return: Valid url of selected source.
:rtype: str
"""
current_index = self.city_combo_box.currentIndex()
city_code = self.city_combo_box.itemData(current_index, Qt.UserRole)
source = (self.source).format(city_code=city_code)
return source
def copy_gfx(self):
gfx_dir = os.path.join(self.editor_data_dir, "gfx")
if os.path.isdir(gfx_dir):
shutil.rmtree(gfx_dir)
os.makedirs(gfx_dir)
progress = QProgressDialog("", "Abort", 0, 0, self)
progress.setWindowTitle("Copying GFX...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setMinimumDuration(0)
progress.setValue(0)
progress.setAutoClose(False)
progress.setLabelText("Setting up GFX dir.")
progress.setMaximum(5)
progress.setValue(0)
# Extract the images we can't just take directly from the game's data.
gfx_bin = zipfile.ZipFile("data/gfx_base.bin", "r")
progress.setValue(1)
gfx_enc = gfx_bin.open("gfx_base.bin")
progress.setValue(2)
gfx_dec = cStringIO.StringIO()
base64.decode(gfx_enc, gfx_dec)
progress.setValue(3)
gfx_base = zipfile.ZipFile(gfx_dec, "r")
progress.setValue(4)
gfx_base.extractall(gfx_dir)
progress.setValue(5)
gfx_base.close()
gfx_dec.close()
gfx_enc.close()
gfx_bin.close()
# We can mostly loop this.
gfx_data = [
("ammo", "kotodama_icn_???.gim"),
("bgd", "bgd_???.gim"),
("cutin", "cutin_icn_???.gim"),
("events", "gallery_icn_???.gim"),
("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"),
("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"),
("nametags", "tex_system.pak/00[12]?.gim"),
("nametags", "tex_system.pak/003[0123456].gim"),
("presents", "present_icn_???.gim"),
("sprites", "bustup_??_??.gim"),
("sprites", "stand_??_??.gmo"),
]
for (dir, file_glob) in gfx_data:
out_dir = os.path.join(gfx_dir, dir)
files = glob.glob(os.path.join(self.umdimage_dir, file_glob))
progress.setLabelText("Copying %s." % dir)
progress.setMaximum(len(files))
progress.setValue(0)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(out_dir, os.path.basename(src))
shutil.copy(src, dest)
progress.setValue(len(files))
progress.setLabelText("Copying font.")
progress.setMaximum(4)
progress.setValue(0)
# The font we have to get from umdimage2.
font_dir = os.path.join(gfx_dir, "font")
if not os.path.isdir(font_dir):
os.makedirs(font_dir)
progress.setValue(1)
# And convert to PNG with an alpha channel so our editor can use it.
font1 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0000.bmp"))
progress.setValue(2)
font2 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0002.bmp"))
progress.setValue(3)
font1.save(os.path.join(font_dir, "Font01.png"))
font2.save(os.path.join(font_dir, "Font02.png"))
shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font"))
shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font"))
progress.setValue(4)
# And then the flash files. This'll be fun.
flash_dir = os.path.join(gfx_dir, "flash")
if not os.path.isdir(flash_dir):
os.makedirs(flash_dir)
# Because there's so many in so many different places, I just stored a list
# of the flash files we need in the gfx_base archive. So let's load that.
with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla:
fla_list = fla.readlines()
progress.setLabelText("Copying flash.")
progress.setMaximum(len(fla_list))
progress.setValue(0)
for i, flash in enumerate(fla_list):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
flash = flash.strip()
fla_name = flash[:7] # fla_###
src = os.path.join(self.umdimage_dir, flash)
dest = os.path.join(flash_dir, "%s.gim" % fla_name)
shutil.copy(src, dest)
progress.setValue(len(fla_list))
# We have a couple sets of files that aren't named the way we want them to
# be, just because of how they're stored in umdimage.
progress.setLabelText("Renaming files.")
to_rename = [
("movies", "movie_%03d.gim", range(32)),
("nametags", "%02d.gim", range(23) + [24, 25, 30, 31]),
]
for (folder, pattern, nums) in to_rename:
folder = os.path.join(gfx_dir, folder)
files = glob.glob(os.path.join(folder, "*.gim"))
progress.setMaximum(len(files))
progress.setValue(0)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(folder, pattern % nums[i])
if os.path.isfile(dest):
os.remove(dest)
shutil.move(src, dest)
sprite_dir = os.path.join(gfx_dir, "sprites")
gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo"))
progress.setLabelText("Extracting GMO files.")
progress.setValue(0)
progress.setMaximum(len(gmo_files))
for i, gmo_file in enumerate(gmo_files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
name, ext = os.path.splitext(os.path.basename(gmo_file))
gim_file = os.path.join(sprite_dir, name + ".gim")
gmo = GmoFile(filename = gmo_file)
# Once we've loaded it, we're all done with it, so make it go away.
os.remove(gmo_file)
if gmo.gim_count() == 0:
continue
gim = gmo.get_gim(0)
with open(gim_file, "wb") as f:
gim.tofile(f)
if self.ui.chkGimToPng.isChecked():
gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim"))
progress.setLabelText("Converting GIM to PNG.")
progress.setValue(0)
progress.setMaximum(len(gim_files))
converter = GimConverter()
for i, gim_file in enumerate(gim_files):
progress.setValue(i)
if progress.wasCanceled():
return
converter.gim_to_png(gim_file)
os.remove(gim_file)
progress.close()
self.gfx_dir = gfx_dir
self.ui.grpStep5.setEnabled(False)
self.ui.grpStep6.setEnabled(True)
def setup_workspace(self):
data0 = os.path.join(self.iso_dir, DATA0_CPK)
self.generate_directories()
progress = QProgressDialog("", QtCore.QString(), 0, 11000, self)
progress.setWindowTitle("Setting up workspace...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setMinimumDuration(0)
progress.setValue(0)
progress.setAutoClose(False)
progress.setAutoReset(False)
progress.setLabelText("Creating directories...")
# Do the easy stuff first.
if not os.path.isdir(self.changes_dir):
os.makedirs(self.changes_dir)
progress.setValue(progress.value() + 1)
if not os.path.isdir(self.backup_dir):
os.makedirs(self.backup_dir)
progress.setValue(progress.value() + 1)
thread_fns = [{"target": extract_cpk, "kwargs": {"filename": data0, "out_dir": self.data0_dir}}]
# Going to capture stdout because I don't feel like
# rewriting the extract functions to play nice with GUI.
stdout = sys.stdout
sys.stdout = cStringIO.StringIO()
for thread_fn in thread_fns:
thread = threading.Thread(**thread_fn)
thread.start()
while thread.isAlive():
thread.join(THREAD_TIMEOUT)
output = [line for line in sys.stdout.getvalue().split("\n") if len(line) > 0]
progress.setValue(progress.value() + len(output))
if len(output) > 0:
progress.setLabelText("Extracting %s..." % output[-1])
sys.stdout = cStringIO.StringIO()
sys.stdout = stdout
# Give us an ISO directory for the editor to place modified files in.
progress.setLabelText("Copying ISO files...")
# ISO directory needs to not exist for copytree.
if os.path.isdir(self.edited_iso_dir):
shutil.rmtree(self.edited_iso_dir)
# One more thing we want threaded so it doesn't lock up the GUI.
thread = threading.Thread(target=shutil.copytree, kwargs={"src": self.iso_dir, "dst": self.edited_iso_dir})
thread.start()
while thread.isAlive():
thread.join(THREAD_TIMEOUT)
progress.setLabelText("Copying ISO files...")
# It has to increase by some amount or it won't update and the UI will lock up.
progress.setValue(progress.value() + 1)
# shutil.copytree(self.iso_dir, self.edited_iso_dir)
progress.setValue(progress.value() + 1)
# Files we want to make blank, because they're unnecessary.
blank_files = [
os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "DATA.BIN"),
os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "EBOOT.BIN"),
os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "PARAM.SFO"),
]
for blank in blank_files:
with open(blank, "wb") as f:
pass
# Copy the decrypted EBOOT into the ISO folder and apply our hacks to it.
progress.setLabelText("Hacking EBOOT...")
progress.setValue(progress.value() + 1)
hacked_eboot = BitStream(filename=self.eboot_path)
hacked_eboot = apply_eboot_patches(hacked_eboot)
with open(os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"), "wb") as f:
hacked_eboot.tofile(f)
# shutil.copy(self.eboot_path, os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"))
progress.setLabelText("Extracting editor data...")
progress.setValue(progress.value() + 1)
# Extract the editor data.
editor_data = zipfile.ZipFile("data/editor_data.zip", "r")
editor_data.extractall(self.editor_data_dir)
editor_data.close()
progress.setValue(progress.maximum())
progress.close()
self.ui.grpStep4.setEnabled(False)
self.ui.grpStep5.setEnabled(True)
def import_umdimage2(src, dst, convert_png=True, propogate=True, parent=None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError(
"Cannot import %s. Source and destination directories are the same."
% src)
answer = QtGui.QMessageBox.question(
parent,
"Import Directory",
"Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" +
dst + "\n\n" + "Any affected files will be backed up. Proceed?",
buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton=QtGui.QMessageBox.No)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Importing...", "Cancel", 0, 0, parent)
progress.setWindowTitle("Importing...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
tmp_dst = tempfile.mkdtemp(prefix="sdse-")
backup_dir = None
for pak in glob.iglob(os.path.join(src, "bg_*.pak")):
if progress.wasCanceled():
break
pak_name = os.path.basename(pak)
backup_dir = backup_files(dst, [pak_name],
suffix="_IMPORT",
backup_dir=backup_dir)
# If we have a regular file with the bg_*.pak name, then just drop it in.
if os.path.isfile(pak):
progress.setLabelText("Importing:\n" + pak_name)
progress.setValue(progress.value() ^ 1)
shutil.copy2(pak, os.path.join(dst, pak_name))
# Otherwise, if it's a directory, insert all the textures we find
# into the target bg_*.pak file.
elif os.path.isdir(pak):
for image in list_all_files(pak):
if progress.wasCanceled():
break
ext = os.path.splitext(image)[1].lower()
if ext == ".png" and not convert_png:
continue
base_name = image[len(src) + 1:]
dst_files = []
if propogate:
dupe_name = os.path.splitext(base_name)[0] + ".gim"
dupe_name = os.path.join("umdimage2", dupe_name)
dupe_name = os.path.normpath(os.path.normcase(dupe_name))
dupes = _DUPE_DB.files_in_same_group(dupe_name)
if dupes == None:
dupes = [dupe_name]
for dupe in dupes:
dst_file = dupe[10:] # chop off the "umdimage2/"
dst_file = os.path.splitext(
dst_file)[0] + ext # original extension
dst_file = os.path.join(tmp_dst, dst_file)
dst_files.append(dst_file)
else:
dst_files = [os.path.join(tmp_dst, base_name)]
for dst_file in dst_files:
try:
os.makedirs(os.path.dirname(dst_file))
except:
pass
shutil.copy(image, dst_file)
if progress.wasCanceled():
break
progress.setLabelText("Inserting textures into:\n" + pak_name)
progress.setValue(progress.value() ^ 1)
pak_dir = os.path.join(tmp_dst, pak_name)
pak_file = os.path.join(dst, pak_name)
# If we didn't copy anything over, just move on.
if not os.path.isdir(pak_dir):
continue
thread = threading.Thread(target=insert_textures,
args=(pak_dir, pak_file))
thread.start()
while thread.isAlive():
thread.join(MIN_INTERVAL)
progress.setValue(progress.value() ^ 1)
if progress.wasCanceled():
progress.setLabelText("Canceling...")
shutil.rmtree(tmp_dst)
progress.close()
def copy_gfx(self):
gfx_dir = os.path.join(self.editor_data_dir, "gfx")
if os.path.isdir(gfx_dir):
shutil.rmtree(gfx_dir)
os.makedirs(gfx_dir)
progress = QProgressDialog("", "Abort", 0, 0, self)
progress.setWindowTitle("Copying GFX...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setMinimumDuration(0)
progress.setValue(0)
progress.setAutoClose(False)
progress.setLabelText("Setting up GFX dir.")
progress.setMaximum(5)
progress.setValue(0)
# Extract the images we can't just take directly from the game's data.
gfx_bin = zipfile.ZipFile("data/gfx_base.zip", "r")
progress.setValue(1)
progress.setValue(2)
gfx_bin.extractall(gfx_dir)
progress.setValue(5)
gfx_bin.close()
# We can mostly loop this.
gfx_data = [
("ammo", "kotodama_icn_???.gim"),
("bgd", "bgd_???.gim"),
("cutin", "cutin_icn_???.gim"),
("events", "gallery_icn_???.gim"),
("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"),
("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"),
("movies", "gallery_ico_m_none.gim"),
("movies", "gallery_thumbnail_m_???.gim"),
("nametags", "tex_system.pak/00[12]?.gim"),
("nametags", "tex_system.pak/003[0123456].gim"),
("presents", "present_icn_???.gim"),
("sprites", "bustup_??_??.gim"),
("sprites", "stand_??_??.gmo"),
]
for (dir, file_glob) in gfx_data:
out_dir = os.path.join(gfx_dir, dir)
files = glob.glob(os.path.join(self.data0_dir, file_glob))
progress.setLabelText("Copying %s." % dir)
progress.setMaximum(len(files))
progress.setValue(0)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(out_dir, os.path.basename(src))
shutil.copy(src, dest)
progress.setValue(len(files))
progress.setLabelText("Copying font.")
progress.setMaximum(4)
progress.setValue(0)
# The font we have to get from umdimage2.
font_dir = os.path.join(gfx_dir, "font")
if not os.path.isdir(font_dir):
os.makedirs(font_dir)
progress.setValue(1)
# And convert to PNG with an alpha channel so our editor can use it.
font1 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0000.bmp"))
progress.setValue(2)
font2 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0002.bmp"))
progress.setValue(3)
font1.save(os.path.join(font_dir, "Font01.png"))
font2.save(os.path.join(font_dir, "Font02.png"))
shutil.copy(
os.path.join(self.data01_dir, "jp", "font", "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font")
)
shutil.copy(
os.path.join(self.data01_dir, "jp", "font", "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font")
)
progress.setValue(4)
# And then the flash files. This'll be fun.
flash_dir = os.path.join(gfx_dir, "flash")
if not os.path.isdir(flash_dir):
os.makedirs(flash_dir)
# Because there's so many in so many different places, I just stored a list
# of the flash files we need in the gfx_base archive. So let's load that.
with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla:
fla_list = fla.readlines()
progress.setLabelText("Copying flash.")
progress.setMaximum(len(fla_list))
progress.setValue(0)
for i, flash in enumerate(fla_list):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
flash = flash.strip()
fla_name = flash[:7] # fla_###
src = os.path.join(self.data01_dir, "all", "flash", flash)
dest = os.path.join(flash_dir, "%s.gim" % fla_name)
shutil.copy(src, dest)
progress.setValue(len(fla_list))
# We have a couple sets of files that aren't named the way we want them to
# be, just because of how they're stored in umdimage.
progress.setLabelText("Renaming files.")
to_rename = [("movies", "movie_%03d.gim", range(32)), ("nametags", "%02d.gim", range(23) + [24, 25, 30, 31])]
for (folder, pattern, nums) in to_rename:
folder = os.path.join(gfx_dir, folder)
files = glob.glob(os.path.join(folder, "*.gim"))
progress.setMaximum(len(files))
progress.setValue(0)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(folder, pattern % nums[i])
if os.path.isfile(dest):
os.remove(dest)
shutil.move(src, dest)
sprite_dir = os.path.join(gfx_dir, "sprites")
gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo"))
progress.setLabelText("Extracting GMO files.")
progress.setValue(0)
progress.setMaximum(len(gmo_files))
for i, gmo_file in enumerate(gmo_files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
name, ext = os.path.splitext(os.path.basename(gmo_file))
gim_file = os.path.join(sprite_dir, name + ".gim")
gmo = GmoFile(filename=gmo_file)
# Once we've loaded it, we're all done with it, so make it go away.
os.remove(gmo_file)
if gmo.gim_count() == 0:
continue
gim = gmo.get_gim(0)
with open(gim_file, "wb") as f:
gim.tofile(f)
if self.ui.chkGimToPng.isChecked():
gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim"))
progress.setLabelText("Converting GIM to PNG.")
progress.setValue(0)
progress.setMaximum(len(gim_files))
converter = GimConverter()
for i, gim_file in enumerate(gim_files):
progress.setValue(i)
if progress.wasCanceled():
return
converter.gim_to_png(gim_file)
os.remove(gim_file)
progress.close()
self.gfx_dir = gfx_dir
self.ui.grpStep5.setEnabled(False)
self.ui.grpStep6.setEnabled(True)
def export_umdimage2(src, dst, convert_gim=True, unique=False, parent=None):
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.normcase(src) == os.path.normcase(dst):
raise ValueError(
"Cannot export %s. Source and destination directories are the same."
% src)
answer = QtGui.QMessageBox.question(
parent,
"Export Directory",
"Exporting directory:\n\n" + src + "\n\n" + "into directory:\n\n" +
dst + "\n\n" + "Proceed?",
buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
defaultButton=QtGui.QMessageBox.No)
if answer == QtGui.QMessageBox.No:
return
progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent)
progress.setWindowTitle("Exporting...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setValue(0)
progress.setAutoClose(False)
progress.setMinimumDuration(0)
if unique:
tmp_dst = tempfile.mkdtemp(prefix="sdse-")
else:
tmp_dst = dst
seen_groups = []
for pak in glob.iglob(os.path.join(src, "bg_*.pak")):
if progress.wasCanceled():
break
pak_name = os.path.basename(pak)
out_dir = os.path.join(tmp_dst, pak_name)
progress.setLabelText("Extracting:\n" + pak)
thread = threading.Thread(target=extract_model_pak,
args=(pak, out_dir, convert_gim))
thread.start()
while thread.isAlive():
thread.join(MIN_INTERVAL)
progress.setValue(progress.value() ^ 1)
if progress.wasCanceled():
progress.setLabelText("Canceling...")
if progress.wasCanceled():
break
if unique:
for img in list_all_files(out_dir):
img_base = img[len(tmp_dst) + 1:]
dupe_name = os.path.splitext(img_base)[0] + ".gim"
dupe_name = os.path.join("umdimage2", dupe_name)
dupe_name = os.path.normpath(os.path.normcase(dupe_name))
group = _DUPE_DB.group_from_file(dupe_name)
if group in seen_groups:
continue
if not group == None:
seen_groups.append(group)
dst_file = os.path.join(dst, img_base)
dst_dir = os.path.dirname(dst_file)
try:
os.makedirs(dst_dir)
except:
pass
shutil.copy2(img, dst_file)
shutil.rmtree(out_dir)
if unique:
shutil.rmtree(tmp_dst)
progress.close()
class ImportDialog(QDialog, Ui_ImportDialogBase):
def __init__(self, theParent=None, theIface=None):
"""Constructor for import dialog.
Args:
* theParent - Optional widget to use as parent
* theIface - an instance of QGisInterface
Returns:
not applicable
Raises:
no exceptions explicitly raised
"""
QDialog.__init__(self, theParent)
self.parent = theParent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = theIface
self.url = "http://osm.linfiniti.com/buildings-shp"
## region coordinate: (latitude, longtitude, zoom_level)
self.regionExtent = {
'0': [18.87685, -71.493, 6], # haiti
'1': [-2.5436300, 116.8887, 3], # indonesia
'2': [1.22449, 15.40999, 2], # africa
'3': [34.05, 56.55, 3], # middle east
'4': [12.98855, 121.7166, 4], # philipine
}
# creating progress dialog for download
self.progressDialog = QProgressDialog(self)
self.progressDialog.setAutoClose(False)
myTitle = self.tr("InaSAFE OpenStreetMap Downloader")
self.progressDialog.setWindowTitle(myTitle)
## set map parameter based on placeholder self.map widget
theMap = InasafeLightMaps(self.gbxMap)
theMap.setGeometry(self.map.geometry())
theMap.setSizePolicy(self.map.sizePolicy())
self.map = theMap
self.nam = QNetworkAccessManager(self)
self.setupOptions()
self.restoreState()
self.cbxRegion.currentIndexChanged.connect(self.regionChanged)
self.map.m_normalMap.updated.connect(self.updateExtent)
def regionChanged(self, theIndex):
""" Slot that called when region combo box changed.
Params:
theIndex - index of combo box
"""
myRegionIndex = str(self.cbxRegion.itemData(theIndex).toString())
myCenter = self.regionExtent[myRegionIndex]
self.map.setCenter(myCenter[0], myCenter[1], myCenter[2])
# pylint: disable=W0613
def resizeEvent(self, theEvent):
""" Function that called when resize event occurred.
Params:
theEvent - QEvent instance. Not used.
"""
self.map.resize(self.gbxMap.width() - 30, self.gbxMap.height() - 30)
# pylint: disable=W0613
def setupOptions(self):
""" Set the content of combo box for region and preset """
self.cbxRegion.insertItem(0, 'Indonesia', 1)
self.cbxRegion.insertItem(1, 'Africa', 2)
self.cbxRegion.insertItem(2, 'Philippines', 4)
self.cbxRegion.insertItem(3, 'Central Asia/Middle East', 3)
self.cbxRegion.insertItem(4, 'Haiti', 0)
self.cbxPreset.insertItem(0, self.tr('Buildings'), 'building')
self.cbxPreset.insertItem(0, self.tr('Highway'), 'highway')
def restoreState(self):
""" Read last state of GUI from configuration file """
mySetting = QSettings()
myRegion = mySetting.value('region').toInt()
if myRegion[1] is True:
self.cbxRegion.setCurrentIndex(myRegion[0])
myPreset = mySetting.value('preset').toInt()
if myPreset[1] is True:
self.cbxPreset.setCurrentIndex(myPreset[0])
self.outDir.setText(mySetting.value('directory').toString())
myZoomLevel = mySetting.value('zoom_level').toInt()
myLatitude = mySetting.value('latitude').toDouble()
myLongitude = mySetting.value('longitude').toDouble()
if myZoomLevel[1] is True:
self.map.setCenter(myLatitude[0], myLongitude[0], myZoomLevel[0])
else:
# just set to indonesia extent
myCenter = self.regionExtent['1']
self.map.setCenter(myCenter[0], myCenter[1], myCenter[2])
def saveState(self):
""" Store current state of GUI to configuration file """
mySetting = QSettings()
mySetting.setValue('region', self.cbxRegion.currentIndex())
mySetting.setValue('preset', self.cbxPreset.currentIndex())
mySetting.setValue('directory', self.outDir.text())
mySetting.setValue('zoom_level', self.map.getZoomLevel())
myCenter = self.map.getCenter()
mySetting.setValue('latitude', myCenter[0])
mySetting.setValue('longitude', myCenter[1])
def updateExtent(self):
""" Update extent value in GUI based from value in map widget"""
myExtent = self.map.getExtent()
self.minLongitude.setText(str(myExtent[1]))
self.minLatitude.setText(str(myExtent[0]))
self.maxLongitude.setText(str(myExtent[3]))
self.maxLatitude.setText(str(myExtent[2]))
@pyqtSignature('') # prevents actions being handled twice
def on_pBtnDir_clicked(self):
""" Show a dialog to choose directory """
self.outDir.setText(QFileDialog.getExistingDirectory(
self, self.tr("Select Directory")))
def accept(self):
""" Do import process """
try:
self.saveState()
self.ensureDirExist()
self.doImport()
self.loadShapeFile()
self.done(QDialog.Accepted)
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as myEx:
QMessageBox.warning(
self,
self.tr("InaSAFE OpenStreetMap Downloader Error"),
str(myEx))
self.progressDialog.cancel()
def ensureDirExist(self):
"""
Ensure directory path entered in dialog exist.
When the path is not exist, this function will
ask the user if he want to create it or not.
Raises:
CanceledImportDialogError - when user choose 'No'
in question dialog for creating directory.
"""
myDir = str(self.outDir.text())
if os.path.exists(myDir):
return
myTitle = self.tr("Directory %1 not exist").arg(myDir)
myQuestion = self.tr(
"Directory %1 not exist. Are you want to create it?"
).arg(myDir)
myAnswer = QMessageBox.question(
self, myTitle,
myQuestion, QMessageBox.Yes | QMessageBox.No)
if myAnswer == QMessageBox.Yes:
os.makedirs(myDir)
else:
raise CanceledImportDialogError()
def doImport(self):
"""
Import shape files from Linfinti.
Raises:
* ImportDialogError - when network error occurred
* CanceledImportDialogError - when user press cancel button
"""
## preparing necessary data
myMinLng = str(self.minLongitude.text())
myMinLat = str(self.minLatitude.text())
myMaxLng = str(self.maxLongitude.text())
myMaxLat = str(self.maxLatitude.text())
myCurrentIndex = self.cbxPreset.currentIndex()
myType = str(self.cbxPreset.itemData(myCurrentIndex).toString())
myCoordinate = "{myMinLng},{myMinLat},{myMaxLng},{myMaxLat}".format(
myMinLng=myMinLng,
myMinLat=myMinLat,
myMaxLng=myMaxLng,
myMaxLat=myMaxLat
)
myShapeUrl = "{url}?bbox={myCoordinate}&obj={type}".format(
url=self.url,
myCoordinate=myCoordinate,
type=myType
)
myFilePath = tempfile.mktemp('.shp.zip')
# download and extract it
self.downloadShapeFile(myShapeUrl, myFilePath)
self.extractZip(myFilePath, str(self.outDir.text()))
self.progressDialog.done(QDialog.Accepted)
def downloadShapeFile(self, theUrl, theOutput):
"""
Download shape file from theUrl and write to theOutput.
Params:
* theUrl - URL of shape file
* theOutput - path of output file
Raises:
* ImportDialogError - when network error occurred
"""
self.progressDialog.show()
self.progressDialog.setMaximum(100)
self.progressDialog.setValue(0)
# myLabelText = "Begin downloading shapefile from " \
# + "%1 ..."
# self.progressDialog.setLabelText(self.tr(myLabelText).arg(theUrl))
myLabelText = self.tr("Begin downloading shapefile")
self.progressDialog.setLabelText(myLabelText)
myResult = httpDownload(self.nam, theUrl, theOutput,
self.progressDialog)
if myResult is not True:
_, myErrorMessage = myResult
raise ImportDialogError(myErrorMessage)
def extractZip(self, thePath, theOutDir):
"""
Extract all content of zip file from thePath to theOutDir.
Args:
* thePath - the path of zip file
* theOutDir - output directory
Raises:
IOError - when cannot open thePath or theOutDir is not exist.
"""
import zipfile
## extract all files...
myHandle = open(thePath, 'rb')
myZip = zipfile.ZipFile(myHandle)
for myName in myZip.namelist():
myOutPath = os.path.join(theOutDir, myName)
myOutFile = open(myOutPath, 'wb')
myOutFile.write(myZip.read(myName))
myOutFile.close()
myHandle.close()
def loadShapeFile(self):
"""
Load downloaded shape file to QGIS Main Window.
Raises:
ImportDialogError - when buildings.shp not exist
"""
myDir = str(self.outDir.text())
myPath = os.path.join(myDir, 'buildings.shp')
if not os.path.exists(myPath):
myMessage = self.tr(
"%s don't exist. The server don't have buildings data."
)
raise ImportDialogError(myMessage)
self.iface.addVectorLayer(myPath, 'buildings', 'ogr')
class OsmDownloaderDialog(QDialog, FORM_CLASS):
"""Downloader for OSM data."""
def __init__(self, parent=None, iface=None):
"""Constructor for import dialog.
:param parent: Optional widget to use as parent
:type parent: QWidget
:param iface: An instance of QGisInterface
:type iface: QGisInterface
"""
QDialog.__init__(self, parent)
self.parent = parent
self.setupUi(self)
self.setWindowTitle(self.tr('InaSAFE OpenStreetMap Downloader'))
self.iface = iface
self.help_context = 'openstreetmap_downloader'
# creating progress dialog for download
self.progress_dialog = QProgressDialog(self)
self.progress_dialog.setAutoClose(False)
title = self.tr('InaSAFE OpenStreetMap Downloader')
self.progress_dialog.setWindowTitle(title)
# Set up context help
self.help_button = self.button_box.button(QtGui.QDialogButtonBox.Help)
# Allow toggling the help button
self.help_button.setCheckable(True)
self.help_button.toggled.connect(self.help_toggled)
self.stacked_widget.setCurrentIndex(1)
# Disable boundaries group box until boundary checkbox is ticked
self.boundary_group.setEnabled(False)
# set up the validator for the file name prefix
expression = QRegExp('^[A-Za-z0-9-_]*$')
validator = QRegExpValidator(expression, self.filename_prefix)
self.filename_prefix.setValidator(validator)
self.restore_state()
# Setup the rectangle map tool
self.canvas = iface.mapCanvas()
self.rectangle_map_tool = \
RectangleMapTool(self.canvas)
self.rectangle_map_tool.rectangle_created.connect(
self.update_extent_from_rectangle)
self.capture_button.clicked.connect(
self.drag_rectangle_on_map_canvas)
# Setup pan tool
self.pan_tool = QgsMapToolPan(self.canvas)
self.canvas.setMapTool(self.pan_tool)
# Setup helper for admin_level
json_file_path = resources_path('osm', 'admin_level_per_country.json')
if os.path.isfile(json_file_path):
self.countries = json.load(open(json_file_path))
self.bbox_countries = None
self.populate_countries()
# connect
self.country_comboBox.currentIndexChanged.connect(
self.update_helper_political_level)
self.admin_level_comboBox.currentIndexChanged.connect(
self.update_helper_political_level)
self.update_extent_from_map_canvas()
def update_helper_political_level(self):
"""To update the helper about the country and the admin_level."""
current_country = self.country_comboBox.currentText()
index = self.admin_level_comboBox.currentIndex()
current_level = self.admin_level_comboBox.itemData(index)
content = None
try:
content = \
self.countries[current_country]['levels'][str(current_level)]
if content == 'N/A' or content == 'fixme' or content == '':
raise KeyError
except KeyError:
content = self.tr('undefined')
finally:
text = self.tr('which represents %s in') % (content)
self.boundary_helper.setText(text)
def populate_countries(self):
"""Populate the combobox about countries and levels."""
for i in range(1, 12):
self.admin_level_comboBox.addItem(self.tr("Level %s" % i), i)
# Set current index to admin_level 8, the most common one
self.admin_level_comboBox.setCurrentIndex(7)
list_countries = self.countries.keys()
list_countries.sort()
for country in list_countries:
self.country_comboBox.addItem(country)
self.bbox_countries = {}
for country in list_countries:
coords = self.countries[country]['bbox']
self.bbox_countries[country] = QgsRectangle(
coords[0], coords[3], coords[2], coords[1])
self.update_helper_political_level()
@pyqtSlot()
@pyqtSignature('bool') # prevents actions being handled twice
def help_toggled(self, flag):
"""Show or hide the help tab in the stacked widget.
..versionadded: 3.2
:param flag: Flag indicating whether help should be shown or hidden.
:type flag: bool
"""
if flag:
self.help_button.setText(self.tr('Hide Help'))
self.show_help()
else:
self.help_button.setText(self.tr('Show Help'))
self.hide_help()
def hide_help(self):
"""Hide the usage info from the user.
.. versionadded:: 3.2
"""
self.stacked_widget.setCurrentIndex(1)
def show_help(self):
"""Show usage info to the user."""
# Read the header and footer html snippets
self.stacked_widget.setCurrentIndex(0)
header = html_header()
footer = html_footer()
string = header
heading = m.Heading(self.tr('OSM Downloader'), **INFO_STYLE)
body = self.tr(
'This tool will fetch building (\'structure\') or road ('
'\'highway\') data from the OpenStreetMap project for you. '
'The downloaded data will have InaSAFE keywords defined and a '
'default QGIS style applied. To use this tool effectively:'
)
tips = m.BulletedList()
tips.add(self.tr(
'Your current extent, when opening this window, will be used to '
'determine the area for which you want data to be retrieved.'
'You can interactively select the area by using the '
'\'select on map\' button - which will temporarily hide this '
'window and allow you to drag a rectangle on the map. After you '
'have finished dragging the rectangle, this window will '
'reappear.'))
tips.add(self.tr(
'Check the output directory is correct. Note that the saved '
'dataset will be called either roads.shp or buildings.shp (and '
'associated files).'
))
tips.add(self.tr(
'By default simple file names will be used (e.g. roads.shp, '
'buildings.shp). If you wish you can specify a prefix to '
'add in front of this default name. For example using a prefix '
'of \'padang-\' will cause the downloaded files to be saved as '
'\'padang-roads.shp\' and \'padang-buildings.shp\'. Note that '
'the only allowed prefix characters are A-Z, a-z, 0-9 and the '
'characters \'-\' and \'_\'. You can leave this blank if you '
'prefer.'
))
tips.add(self.tr(
'If a dataset already exists in the output directory it will be '
'overwritten.'
))
tips.add(self.tr(
'This tool requires a working internet connection and fetching '
'buildings or roads will consume your bandwidth.'))
tips.add(m.Link(
'http://www.openstreetmap.org/copyright',
text=self.tr(
'Downloaded data is copyright OpenStreetMap contributors'
' (click for more info).')
))
message = m.Message()
message.add(m.Brand())
message.add(heading)
message.add(body)
message.add(tips)
string += message.to_html()
string += footer
self.web_view.setHtml(string)
def restore_state(self):
""" Read last state of GUI from configuration file."""
settings = QSettings()
try:
last_path = settings.value('directory', type=str)
except TypeError:
last_path = ''
self.output_directory.setText(last_path)
def save_state(self):
""" Store current state of GUI to configuration file """
settings = QSettings()
settings.setValue('directory', self.output_directory.text())
def update_extent(self, extent):
"""Update extent value in GUI based from an extent.
:param extent: A list in the form [xmin, ymin, xmax, ymax] where all
coordinates provided are in Geographic / EPSG:4326.
:type extent: list
"""
self.x_minimum.setValue(extent[0])
self.y_minimum.setValue(extent[1])
self.x_maximum.setValue(extent[2])
self.y_maximum.setValue(extent[3])
# Updating the country if possible.
rectangle = QgsRectangle(extent[0], extent[1], extent[2], extent[3])
center = rectangle.center()
for country in self.bbox_countries:
if self.bbox_countries[country].contains(center):
index = self.country_comboBox.findText(country)
self.country_comboBox.setCurrentIndex(index)
break
else:
self.country_comboBox.setCurrentIndex(0)
def update_extent_from_map_canvas(self):
"""Update extent value in GUI based from value in map.
.. note:: Delegates to update_extent()
"""
self.bounding_box_group.setTitle(
self.tr('Bounding box from the map canvas'))
# Get the extent as [xmin, ymin, xmax, ymax]
extent = viewport_geo_array(self.iface.mapCanvas())
self.update_extent(extent)
def update_extent_from_rectangle(self):
"""Update extent value in GUI based from the QgsMapTool rectangle.
.. note:: Delegates to update_extent()
"""
self.show()
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.canvas.setMapTool(self.pan_tool)
rectangle = self.rectangle_map_tool.rectangle()
if rectangle:
self.bounding_box_group.setTitle(
self.tr('Bounding box from rectangle'))
extent = rectangle_geo_array(rectangle, self.iface.mapCanvas())
self.update_extent(extent)
@pyqtSignature('') # prevents actions being handled twice
def on_directory_button_clicked(self):
"""Show a dialog to choose directory."""
# noinspection PyCallByClass,PyTypeChecker
self.output_directory.setText(QFileDialog.getExistingDirectory(
self, self.tr('Select download directory')))
def drag_rectangle_on_map_canvas(self):
"""Hide the dialog and allow the user to draw a rectangle."""
self.hide()
self.rectangle_map_tool.reset()
self.canvas.unsetMapTool(self.pan_tool)
self.canvas.setMapTool(self.rectangle_map_tool)
def get_checked_features(self):
"""Create a tab with all checked features.
:return A list with all features which are checked in the UI.
:rtype list
"""
feature_types = []
if self.roads_flag.isChecked():
feature_types.append('roads')
if self.buildings_flag.isChecked():
feature_types.append('buildings')
if self.building_points_flag.isChecked():
feature_types.append('building-points')
if self.potential_idp_flag.isChecked():
feature_types.append('potential-idp')
if self.boundary_flag.isChecked():
level = self.admin_level_comboBox.currentIndex() + 1
feature_types.append('boundary-%s' % level)
return feature_types
def accept(self):
"""Do osm download and display it in QGIS."""
error_dialog_title = self.tr('InaSAFE OpenStreetMap Downloader Error')
# Lock the bounding_box_group
self.bounding_box_group.setDisabled(True)
# Get the extent
y_minimum = self.y_minimum.value()
y_maximum = self.y_maximum.value()
x_minimum = self.x_minimum.value()
x_maximum = self.x_maximum.value()
extent = [x_minimum, y_minimum, x_maximum, y_maximum]
# Validate extent
valid_flag = validate_geo_array(extent)
if not valid_flag:
message = self.tr(
'The bounding box is not valid. Please make sure it is '
'valid or check your projection!')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the bounding_box_group
self.bounding_box_group.setEnabled(True)
return
# Validate features
feature_types = self.get_checked_features()
if len(feature_types) < 1:
message = self.tr(
'No feature selected. '
'Please make sure you have checked one feature.')
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(self, error_dialog_title, message)
# Unlock the bounding_box_group
self.bounding_box_group.setEnabled(True)
return
try:
self.save_state()
self.require_directory()
for feature_type in feature_types:
output_directory = self.output_directory.text()
output_prefix = self.filename_prefix.text()
overwrite = self.overwrite_flag.isChecked()
output_base_file_path = self.get_output_base_path(
output_directory, output_prefix, feature_type, overwrite)
download(
feature_type,
output_base_file_path,
extent,
self.progress_dialog)
try:
self.load_shapefile(feature_type, output_base_file_path)
except FileMissingError as exception:
display_warning_message_box(
self,
error_dialog_title,
exception.message)
self.done(QDialog.Accepted)
self.rectangle_map_tool.reset()
except CanceledImportDialogError:
# don't show anything because this exception raised
# when user canceling the import process directly
pass
except Exception as exception: # pylint: disable=broad-except
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self, error_dialog_title, exception.message)
self.progress_dialog.cancel()
finally:
# Unlock the bounding_box_group
self.bounding_box_group.setEnabled(True)
def get_output_base_path(
self,
output_directory,
output_prefix,
feature_type,
overwrite):
"""Get a full base name path to save the shapefile.
:param output_directory: The directory where to put results.
:type output_directory: str
:param output_prefix: The prefix to add for the shapefile.
:type output_prefix: str
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param overwrite: Boolean to know if we can overwrite existing files.
:type overwrite: bool
:return: The base path.
:rtype: str
"""
path = os.path.join(
output_directory, '%s%s' % (output_prefix, feature_type))
if overwrite:
# If a shapefile exists, we must remove it (only the .shp)
shp = '%s.shp' % path
if os.path.isfile(shp):
os.remove(shp)
else:
separator = '-'
suffix = self.get_unique_file_path_suffix(
'%s.shp' % path, separator)
if suffix:
path = os.path.join(output_directory, '%s%s%s%s' % (
output_prefix, feature_type, separator, suffix))
return path
@staticmethod
def get_unique_file_path_suffix(file_path, separator='-', i=0):
"""Return the minimum number to suffix the file to not overwrite one.
Example : /tmp/a.txt exists.
- With file_path='/tmp/b.txt' will return 0.
- With file_path='/tmp/a.txt' will return 1 (/tmp/a-1.txt)
:param file_path: The file to check.
:type file_path: str
:param separator: The separator to add before the prefix.
:type separator: str
:param i: The minimum prefix to check.
:type i: int
:return: The minimum prefix you should add to not overwrite a file.
:rtype: int
"""
basename = os.path.splitext(file_path)
if i != 0:
file_path_test = os.path.join(
'%s%s%s%s' % (basename[0], separator, i, basename[1]))
else:
file_path_test = file_path
if os.path.isfile(file_path_test):
return OsmDownloaderDialog.get_unique_file_path_suffix(
file_path, separator, i + 1)
else:
return i
def require_directory(self):
"""Ensure directory path entered in dialog exist.
When the path does not exist, this function will ask the user if he
want to create it or not.
:raises: CanceledImportDialogError - when user choose 'No' in
the question dialog for creating directory.
"""
path = self.output_directory.text()
if os.path.exists(path):
return
title = self.tr('Directory %s not exist') % path
question = self.tr(
'Directory %s not exist. Do you want to create it?') % path
# noinspection PyCallByClass,PyTypeChecker
answer = QMessageBox.question(
self, title, question, QMessageBox.Yes | QMessageBox.No)
if answer == QMessageBox.Yes:
if len(path) != 0:
os.makedirs(path)
else:
# noinspection PyCallByClass,PyTypeChecker,PyArgumentList
display_warning_message_box(
self,
self.tr('InaSAFE error'),
self.tr('Output directory can not be empty.'))
raise CanceledImportDialogError()
else:
raise CanceledImportDialogError()
def load_shapefile(self, feature_type, base_path):
"""Load downloaded shape file to QGIS Main Window.
:param feature_type: What kind of features should be downloaded.
Currently 'buildings', 'building-points' or 'roads' are supported.
:type feature_type: str
:param base_path: The base path of the shape file (without extension).
:type base_path: str
:raises: FileMissingError - when buildings.shp not exist
"""
path = '%s.shp' % base_path
if not os.path.exists(path):
message = self.tr(
'%s does not exist. The server does not have any data for '
'this extent.' % path)
raise FileMissingError(message)
self.iface.addVectorLayer(path, feature_type, 'ogr')
canvas_srid = self.canvas.mapRenderer().destinationCrs().srsid()
on_the_fly_projection = self.canvas.hasCrsTransformEnabled()
if canvas_srid != 4326 and not on_the_fly_projection:
if QGis.QGIS_VERSION_INT >= 20400:
self.canvas.setCrsTransformEnabled(True)
else:
display_warning_message_bar(
self.iface,
self.tr('Enable \'on the fly\''),
self.tr(
'Your current projection is different than EPSG:4326. '
'You should enable \'on the fly\' to display '
'correctly your layers')
)
def reject(self):
"""Redefinition of the reject() method
to remove the rectangle selection tool.
It will call the super method.
"""
self.canvas.unsetMapTool(self.rectangle_map_tool)
self.rectangle_map_tool.reset()
super(OsmDownloaderDialog, self).reject()
def ProgressBar(n_steps, label="", allow_cancel=False, parent=None):
"""
Progressbar context manager for showing progress of workflow to user. Example::
with emzed.gui.ProgressBar(n_steps=100, allow_cancel=True) as handler:
for i in range(100):
# we simulate work of step i
# we update progressbar
handler.update(i, "step %03d" % i)
# we can check if user pressed "Cancel" button and stop our "workflow":
if handler.is_canceled():
break
"""
app = guidata.qapplication()
dlg = QProgressDialog(parent)
dlg.setLabelText(label)
dlg.setAutoClose(False)
dlg.setAutoReset(False)
if allow_cancel:
dlg.setCancelButtonText("Cancel")
dlg.setMaximum(n_steps)
class ProgressBarHandler(object):
def __init__(self, n_steps, dlg):
self._dlg = dlg
self._n_steps = n_steps
self._n = 0
self._canceled = False
dlg.canceled.connect(self._set_canceled)
dlg.setValue(0)
def _set_canceled(self):
self._canceled = True
dlg.close()
def update(self, n, message=None):
app.processEvents()
self._n = n
dlg.setValue(n + 1)
if message is not None:
dlg.setLabelText(message)
dlg.update()
app.processEvents()
def is_canceled(self):
return self._canceled
dlg.activateWindow()
dlg.show()
dlg.raise_()
app.processEvents()
handler = ProgressBarHandler(n_steps, dlg)
yield handler
dlg.close()
