def __init__(self, parent=None): """Constructor.""" super(loadMongoDBDialog, self).__init__(parent) # Set up the user interface from Designer. # After setupUI you can access any designer object by doing # self.<objectname>, and you can use autoconnect slots - see # http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html # #widgets-and-dialogs-with-auto-connect QtGui.QDialog.__init__(self, parent, Qt.WindowMinimizeButtonHint) self.ui = Ui_loadMongoDBDialogBase() self.ui.setupUi(self) # sets the progress bar to 0 self.step = 0 self.ui.progressBar.setValue(self.step) # GUI a single click interface loads the on_click_load function self.ui.listCol.itemClicked.connect(self.select_item) # Allows the user to select which attribute they would like to see distinct values for self.ui.view_all.itemClicked.connect(self.row_select) self.ui.load_collection.setEnabled(False)
class loadMongoDBDialog(QtGui.QDialog, FORM_CLASS): def __init__(self, parent=None): """Constructor.""" super(loadMongoDBDialog, self).__init__(parent) # Set up the user interface from Designer. # After setupUI you can access any designer object by doing # self.<objectname>, and you can use autoconnect slots - see # http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html # #widgets-and-dialogs-with-auto-connect QtGui.QDialog.__init__(self, parent, Qt.WindowMinimizeButtonHint) self.ui = Ui_loadMongoDBDialogBase() self.ui.setupUi(self) # sets the progress bar to 0 self.step = 0 self.ui.progressBar.setValue(self.step) # GUI a single click interface loads the on_click_load function self.ui.listCol.itemClicked.connect(self.select_item) # Allows the user to select which attribute they would like to see distinct values for self.ui.view_all.itemClicked.connect(self.row_select) self.ui.load_collection.setEnabled(False) def view_all_attributes(self): # reset the list count and clear the list count = 0 self.ui.view_all.setEnabled(True) self.ui.view_all.clear() data = self.collection.find_one() for key in data.keys(): attribute = QListWidgetItem(key) self.ui.view_all.insertItem(count, attribute) count+=1 def row_select(self, item): self.ui.distinct_button.setEnabled(True) self.selected_attribute = item.text() def view_distinct(self): count = 0 self.ui.view_distinct.setEnabled(True) self.ui.view_distinct.clear() self.list_distinct = self.collection.distinct(self.selected_attribute) # allows loading of non-string types if len(self.list_distinct) != 0: self.data_type = type(self.list_distinct) # converts all attributes to str types self.list_distinct = map(str, self.list_distinct) # limit the number of distinct values we get to see if len(self.list_distinct) > 20: return for key in self.list_distinct: attribute = QListWidgetItem(str(key)) self.ui.view_distinct.insertItem(count, attribute) count+=1 self.ui.set_button.setEnabled(True) def set_attribute(self): self.ui.load_field.clear() self.ui.load_field.addItems(list(set(self.list_distinct))) self.key = self.selected_attribute self.ui.checkBox.setCheckable(True) # this is a GUI function for displaying all collections/ details found in the DB def show_mdb_collection(self, db_name, server_name, geom_name): # reset the list count and clear the list count = 0 self.ui.listCol.clear() self.geom_name = geom_name try: # establish a link to the mongoDB server self.client = MongoClient(str(server_name), 27017, serverSelectionTimeoutMS=500) except: QMessageBox.about(self, "Warning!", "Please select a server to connect to.") return # db name passed on from the user's input self.db = self.client[db_name] try: # create the list of available collections self.collection_list = self.db.collection_names(include_system_collections=False) except: QMessageBox.about(self, "Warning!", "Server might be offline. Check server and try again.") self.reset_all() return # define the number of columns in our tree widget self.ui.listCol.setColumnCount(3) if len(self.collection_list) == 0: return False # search through the list of available collections for collections with geometry for x in self.collection_list: self.data = self.db[x].find_one() num_items = self.db[x].count() try: if self.data[geom_name]: geom_type = self.data[geom_name]['type'] else: geom_type = False except: geom_type = False # details of each collection details = QTreeWidgetItem([str(x), str(geom_type), str(num_items)]) # only display the collections with geometry if geom_type != False: self.ui.listCol.insertTopLevelItem(count, details) count += 1 else: pass self.client.close() return True def reset_all(self): self.ui.distinct_button.setEnabled(False) self.ui.set_button.setEnabled(False) self.ui.view_distinct.setEnabled(False) self.ui.view_all.setEnabled(False) self.ui.load_field.clear() self.ui.checkBox.setCheckable(0) def select_item(self, item, column): self.reset_all() # the name of the collection we will be using self.collection_name = str(item.text(0)) # the geometry of the collection self.geometry_name = str(item.text(1)) # divide the collection count to percentages for the progress bar self.percent = 100/float(item.text(2)) # load the selected collection into a list self.collection = self.db[self.collection_name] # enables the load button when an item is selected from the list self.ui.load_collection.setEnabled(True) self.ui.view_button.setEnabled(True) def on_click_load(self): # default value of the progress bar is set to 0 self.counter = 0 self.ui.progressBar.setValue(self.counter) # check if the user wants to limit the selection by distinct value if self.ui.checkBox.isChecked(): load_key = self.ui.load_field.currentText() # if the data type is not a string if self.data_type != str: load_key = ast.literal_eval(load_key) self.ourList = self.collection.find({self.key: load_key}) else: self.ourList = self.collection.find() # get the first level of attributes, we can modify which fields we want to use self.single_return = self.collection.find_one({}, {'geom':0, 'fibres': 0, 'date': 0}) #{} , {"geom": 1} self.attr_list = self.single_return.keys() # list defined for the attributes table self.attr_list_new = [] # list defined for structural access to tables self.attr_list_structure = [] # locate the sub attributes and store them in sub_attr_list for attr in self.attr_list: # append keys from the first layer self.attr_list_new.append(str(attr)) self.attr_list_structure.append([attr]) # if the key is a dictionary if type(self.single_return[attr]) is dict: # find all of the sub keys in the dictionary sub_keys_list = self.single_return[attr].keys() # search through the list of sub keys for sub_key in sub_keys_list: # structural code for ease of access later struct_sub = [attr],[sub_key] # append the sub keys to our main list, define differences self.attr_list_new.append(str(attr) + "." + str(sub_key)) # create a structure for ease of access self.attr_list_structure.append(struct_sub) else: pass # define the dataLayer type as either Point, LineString or Polygon self.dataLayer = QgsVectorLayer(self.geometry_name, self.collection_name, "memory") # prepare the layer for the data we will be inputing self.dataLayer.startEditing() self.layerData = self.dataLayer.dataProvider() # create the shapefile attributes based on existing mongoDB field for attribute in (self.attr_list_new): self.layerData.addAttributes([ QgsField(attribute, QVariant.String)]) # our attribute container self.feature = QgsFeature() self.feature.initAttributes(len(self.attr_list_new)) for value in self.ourList: #print value[self.geom_name]["type"] # if the user has selected a collection with point geometry if value[self.geom_name]["type"] == "Point": x_coord = value[self.geom_name]["coordinates"][0] y_coord = value[self.geom_name]["coordinates"][1] self.populate_attributes(value) self.feature.setGeometry(QgsGeometry.fromPoint(QgsPoint(x_coord, y_coord))) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) # update the progress bar self.event_progress() self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) elif value[self.geom_name]["type"] == "LineString": # used to store a list of poly lines line_string = [] # checks the geometry and only imports valid geometry if self.check_valid_geom(value): for y in range(len(value[self.geom_name]["coordinates"])): # do not use unless needed, in case there is a multiLineString Object in the DB try: line_string.append(QgsPoint(value[self.geom_name]["coordinates"][y][0], value[self.geom_name]["coordinates"][y][1])) except: qgis.utils.iface.messageBar().pushMessage("Error", "Error loading Linestring on {}: {}".format(str(value["_id"]), str(sys.exc_info()[0])), level=QgsMessageBar.CRITICAL) self.populate_attributes(value) self.feature.setGeometry(QgsGeometry.fromPolyline(line_string)) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) del line_string[:] # update the progress bar self.event_progress() self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) # this deals with Polygon geometry elif value[self.geom_name]["type"] == "Polygon": # store the polygon points poly_shape = [] # store the line (a poly has multiple lines) line_string = [] # checks the geometry and only imports valid geometry if self.check_valid_geom(value): for y in range(len(value[self.geom_name]["coordinates"][0])): try: line_string.append(QgsPoint(value[self.geom_name]["coordinates"][0][y][0], value[self.geom_name]["coordinates"][0][y][1])) except: qgis.utils.iface.messageBar().pushMessage("Error", "Error loading Polygon {}: {}".format(str(value["_id"]), str(sys.exc_info()[0])), level=QgsMessageBar.CRITICAL) poly_shape.append(line_string); self.populate_attributes(value) try: ps = QgsGeometry.fromPolygon(poly_shape) self.feature.setGeometry(ps) except: qgis.utils.iface.messageBar().pushMessage("Error", "Error on {}: {}".format(str(value["_id"]), str(sys.exc_info()[0])), level=QgsMessageBar.CRITICAL) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) del line_string[:] del poly_shape[:] # update the progress bar self.event_progress() self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) # this deals with Polygon geometry elif value[self.geom_name]["type"] == "MultiPolygon": # store the polygon points poly_shape = [] # checks the geometry and only imports valid geometry if self.check_valid_geom(value): multi_poly_shape = [] for multi_shape in value[self.geom_name]["coordinates"]: for shape in multi_shape: each_shape = [] for xy in shape: try: each_shape.append(QgsPoint(xy[0], xy[1])) except: qgis.utils.iface.messageBar().pushMessage("Error", "Error loading Multipolygon {}: {}".format(str(value["_id"]), str(sys.exc_info()[0])), level=QgsMessageBar.CRITICAL) multi_poly_shape.append(each_shape) # final append at highest level poly_shape.append(multi_poly_shape) self.populate_attributes(value) try: ps = QgsGeometry.fromMultiPolygon(poly_shape) self.feature.setGeometry(ps) except: qgis.utils.iface.messageBar().pushMessage("Error", "Error on {}: {}".format(str(value["_id"]), str(sys.exc_info()[0])), level=QgsMessageBar.CRITICAL) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) del poly_shape[:] # update the progress bar self.event_progress() self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) else: qgis.utils.iface.messageBar().pushMessage("Error", "Failed to load geometry due to {} being unsupported".format(value[self.geom_name]["type"]), level=QgsMessageBar.CRITICAL) self.ui.listCol.setEnabled(True) # commits the changes made to the layer and adds the layer to the map self.dataLayer.commitChanges() # the path we will be writing to is the plugin folder dependant on the layer name write_to = str(os.path.abspath(__file__ + "/../../")) + "/" + str(self.collection_name) + ".shp" write_error = QgsVectorFileWriter.writeAsVectorFormat(self.dataLayer, write_to, "system", self.dataLayer.crs(), "ESRI Shapefile") self.dataLayer = QgsVectorLayer(write_to, self.collection_name, "ogr") QgsMapLayerRegistry.instance().addMapLayer(self.dataLayer) # check for valid geometry def check_valid_geom(self, value): if value.has_key(self.geom_name): if value[self.geom_name].has_key("coordinates"): if len(value[self.geom_name]["coordinates"]) != 0: return True return False return False def event_progress(self): # convert to an integer for the progress bar self.counter = self.counter + self.percent self.step = int(self.counter) +1 # set the updated values in the progress bar and process the event self.ui.progressBar.setValue(self.step) QApplication.processEvents() # once the upload has finished, reset the progress back to 0 if self.step >= 100: self.ui.progressBar.setValue(0) # populate keys and sub-keys def populate_attributes(self, value): index_pos = 0 for key in self.attr_list_structure: if len(key) is 1: # this caters for the new data model where the status is a subkey if key[0] == "status": try: stat_list = list(value[str(key[0])]) formatted = stat_list[-1]["label"] self.feature[index_pos] = str(formatted) except: pass index_pos += 1 else: try: self.feature[index_pos] = str(value[str(key[0])]) index_pos += 1 except: index_pos += 1 elif len(key) is 2: try: self.feature[index_pos] = str(value[str(key[0][0])][str(key[1][0])]) index_pos +=1 except: index_pos +=1 else: pass
class loadMongoDBDialog(QtGui.QDialog, FORM_CLASS): def __init__(self, parent=None): """Constructor.""" super(loadMongoDBDialog, self).__init__(parent) # Set up the user interface from Designer. # After setupUI you can access any designer object by doing # self.<objectname>, and you can use autoconnect slots - see # http://qt-project.org/doc/qt-4.8/designer-using-a-ui-file.html # #widgets-and-dialogs-with-auto-connect QtGui.QDialog.__init__(self, parent, Qt.WindowMinimizeButtonHint) self.ui = Ui_loadMongoDBDialogBase() self.ui.setupUi(self) # sets the progress bar to 0 self.step = 0 self.ui.progressBar.setValue(self.step) # GUI a single click interface loads the on_click_load function self.ui.listCol.itemClicked.connect(self.select_item) # Allows the user to select which attribute they would like to see distinct values for self.ui.view_all.itemClicked.connect(self.row_select) self.ui.load_collection.setEnabled(False) def view_all_attributes(self): # reset the list count and clear the list count = 0 self.ui.view_all.setEnabled(True) self.ui.view_all.clear() data = self.collection.find_one() for key in data.keys(): attribute = QListWidgetItem(key) self.ui.view_all.insertItem(count, attribute) count+=1 def row_select(self, item): self.ui.distinct_button.setEnabled(True) self.selected_attribute = item.text() def view_distinct(self): count = 0 self.ui.view_distinct.setEnabled(True) self.ui.view_distinct.clear() self.list_distinct = self.collection.distinct(self.selected_attribute) # allows loading of non-string types if len(self.list_distinct) != 0: self.data_type = type(self.list_distinct) # converts all attributes to str types self.list_distinct = map(str, self.list_distinct) # limit the number of distinct values we get to see if len(self.list_distinct) > 20: return for key in self.list_distinct: attribute = QListWidgetItem(str(key)) self.ui.view_distinct.insertItem(count, attribute) count+=1 self.ui.set_button.setEnabled(True) def set_attribute(self): self.ui.load_field.clear() self.ui.load_field.addItems(list(set(self.list_distinct))) self.key = self.selected_attribute self.ui.checkBox.setCheckable(True) # this is a GUI function for displaying all collections/ details found in the DB def show_mdb_collection(self, db_name, server_name, geom_name): # reset the list count and clear the list count = 0 self.ui.listCol.clear() self.geom_name = geom_name try: # establish a link to the mongoDB server self.client = MongoClient(str(server_name), 27017, serverSelectionTimeoutMS=500) except: QMessageBox.about(self, "Warning!", "Please select a server to connect to.") return # db name passed on from the user's input self.db = self.client[db_name] try: # create the list of available collections self.collection_list = self.db.collection_names(include_system_collections=False) except: QMessageBox.about(self, "Warning!", "Server might be offline. Check server and try again.") self.reset_all() return # define the number of columns in our tree widget self.ui.listCol.setColumnCount(3) if len(self.collection_list) == 0: return False # search through the list of available collections for collections with geometry for x in self.collection_list: self.data = self.db[x].find_one() num_items = self.db[x].count() try: if self.data[geom_name]: geom_type = self.data[geom_name]['type'] else: geom_type = False except: geom_type = False # details of each collection details = QTreeWidgetItem([str(x), str(geom_type), str(num_items)]) # only display the collections with geometry if geom_type != False: self.ui.listCol.insertTopLevelItem(count, details) count += 1 else: pass self.client.close() return True def reset_all(self): self.ui.distinct_button.setEnabled(False) self.ui.set_button.setEnabled(False) self.ui.view_distinct.setEnabled(False) self.ui.view_all.setEnabled(False) self.ui.load_field.clear() self.ui.checkBox.setCheckable(0) def select_item(self, item, column): self.reset_all() # the name of the collection we will be using self.collection_name = str(item.text(0)) # the geometry of the collection self.geometry_name = str(item.text(1)) # divide the collection count to percentages for the progress bar self.percent = 100/float(item.text(2)) self.total = int(item.text(2)) # load the selected collection into a list self.collection = self.db[self.collection_name] # enables the load button when an item is selected from the list self.ui.load_collection.setEnabled(True) self.ui.view_button.setEnabled(True) def on_click_load(self): # default value of the progress bar is set to 0 self.counter = 1 self.ui.progressBar.setValue(self.counter) self.ui.progressBar.setMinimum(1) self.ui.progressBar.setMaximum(self.total) # check if the user wants to limit the selection by distinct value if self.ui.checkBox.isChecked(): load_key = self.ui.load_field.currentText() # if the data type is not a string if self.data_type != str: load_key = ast.literal_eval(load_key) self.ourList = self.collection.find({self.key: load_key}) else: self.ourList = self.collection.find() # get the first level of attributes, we can modify which fields we want to use self.single_return = self.collection.find_one({}, {'geom':0}) #{} , {"geom": 1} self.keys = self.single_return.keys() self.attributes = [] for key in self.keys: if (type(self.single_return[key]) is dict): continue if (type(self.single_return[key]) is list): dtype = QVariant.StringList if (type(self.single_return[key]) is int): dtype = QVariant.Int elif (type(self.single_return[key]) is float): dtype = QVariant.Double else: dtype = QVariant.String # define the attribute dictionnary # add a dictionary of attribute name and type self.attributes.append({"name": str(key), "dtype": dtype, "value": key}) # define the dataLayer type as either Point, LineString or Polygon self.dataLayer = QgsVectorLayer(self.geometry_name + '?crs=EPSG:4326', self.collection_name, "memory") # prepare the layer for the data we will be inputing self.dataLayer.startEditing() self.layerData = self.dataLayer.dataProvider() # create the attributes based on existing mongoDB field # setup the type of each field # for attribute in (self.attr_list_new): for attr in (self.attributes): self.layerData.addAttributes([ QgsField(attr["name"], attr["dtype"])]) # our attribute container self.feature = QgsFeature() self.feature.initAttributes(len(self.attributes)) for value in self.ourList: # if the user has selected a collection with point geometry geom = value[self.geom_name] if geom["type"] == "Point": x_coord = geom["coordinates"][0] y_coord = geom["coordinates"][1] self.populate_attributes(value) self.feature.setGeometry(QgsGeometry.fromPoint(QgsPoint(x_coord, y_coord))) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) if geom["type"] == "LineString": # used to store a list of poly lines line_string = [] # checks the geometry and only imports valid geometry if self.check_valid_geom(value): for y in range(len(geom["coordinates"])): # do not use unless needed, in case there is a multiLineString Object in the DB """ if value["geometry"]["type"] != "LineString": pass else: """ try: line_string.append(QgsPoint(geom["coordinates"][y][0], geom["coordinates"][y][1])) except: print "error on %s: %s" %(str(value["_id"]), str(sys.exc_info()[0])) self.populate_attributes(value) self.feature.setGeometry(QgsGeometry.fromPolyline(line_string)) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) del line_string[:] self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) # this deals with Polygon geometry if geom["type"] == "Polygon": # store the polygon points poly_shape = [] # store the line (a poly has multiple lines) line_string = [] # checks the geometry and only imports valid geometry if self.check_valid_geom(value): for y in range(len(geom["coordinates"][0])): try: line_string.append(QgsPoint(geom["coordinates"][0][y][0], geom["coordinates"][0][y][1])) except: print "error on %s: %s" %(str(value["_id"]), str(sys.exc_info()[0])) poly_shape.append(line_string); self.populate_attributes(value) try: ps = QgsGeometry.fromPolygon(poly_shape) self.feature.setGeometry(ps) except: print "error on %s: %s" %(str(value["_id"]), str(sys.exc_info()[0])) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) del line_string[:] del poly_shape[:] self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) # this deals with Polygon geometry if geom["type"] == "MultiPolygon": # store the polygon points poly_shape = [] # store the line (a poly has multiple lines) line_string = [] # checks the geometry and only imports valid geometry if self.check_valid_geom(value): for polys in range(len(geom["coordinates"])): for y in range(len(geom["coordinates"][polys])): try: line_string.append(QgsPoint(geom["coordinates"][polys][y][0], geom["coordinates"][polys][y][1])) except: print "error on %s: %s" %(str(value["_id"]), str(sys.exc_info()[0])) poly_shape.append(line_string); self.populate_attributes(value) try: ps = QgsGeometry.fromPolygon(poly_shape) self.feature.setGeometry(ps) except: print "error on %s: %s" %(str(value["_id"]), str(sys.exc_info()[0])) (res, outFeats) = self.dataLayer.dataProvider().addFeatures([self.feature]) del line_string[:] del poly_shape[:] self.ui.load_collection.setEnabled(False) self.ui.listCol.setEnabled(False) # update the progress bar self.event_progress() self.ui.listCol.setEnabled(True) # commits the changes made to the layer and adds the layer to the map self.dataLayer.commitChanges() QgsMapLayerRegistry.instance().addMapLayer(self.dataLayer) # check for valid geometry def check_valid_geom(self, value): if value.has_key(self.geom_name): if value[self.geom_name].has_key("type"): if value[self.geom_name].has_key("coordinates"): if len(value[self.geom_name]["coordinates"]) != 0: return True return False return False return False def event_progress(self): # convert to an integer for the progress bar # self.counter = self.counter + self.percent # self.step = int(self.counter) +1 self.step = self.ui.progressBar.value() + 1 # set the updated values in the progress bar and process the event self.ui.progressBar.setValue(self.step) QApplication.processEvents() # once the upload has finished, reset the progress back to 0 # if self.step >= 100: if self.step >= self.total: self.ui.progressBar.setValue(0) # populate keys and sub-keys def populate_attributes(self, value): index_pos = 0 for attr in self.attributes: try: self.feature[index_pos] = value[attr["name"]] except: self.feature[index_pos] = None index_pos += 1