def upload_files(self, layer, field_index, features): """ Upload given features' source files to remote server and return a dict formatted as changeAttributeValues expects to update 'datos' attribute to a remote location. """ if not QSettings().value( 'Asistente-LADM_COL/sources/document_repository', False, bool): self.message_with_duration_emitted.emit( QCoreApplication.translate( "SourceHandler", "The source files were not uploaded to the document repository because you have that option unchecked. You can still upload the source files later using the 'Upload Pending Source Files' menu." ), Qgis.Info, 10) return dict() # Test if we have Internet connection and a valid service dlg = self.qgis_utils.get_settings_dialog() res, msg = dlg.is_source_service_valid() if not res: msg['text'] = QCoreApplication.translate( "SourceHandler", "No file could be uploaded to the document repository. You can do it later from the 'Upload Pending Source Files' menu. Reason: {}" ).format(msg['text']) self.message_with_duration_emitted.emit( msg['text'], Qgis.Info, 20) # The data is still saved, so always show Info msg return dict() file_features = [ feature for feature in features if not feature[field_index] == NULL and os.path.isfile(feature[field_index]) ] total = len(features) not_found = total - len(file_features) upload_dialog = UploadProgressDialog(len(file_features), not_found) upload_dialog.show() count = 0 upload_errors = 0 new_values = dict() for feature in file_features: data_url = feature[field_index] file_name = os.path.basename(data_url) nam = QNetworkAccessManager() #reply.downloadProgress.connect(upload_dialog.update_current_progress) multiPart = QHttpMultiPart(QHttpMultiPart.FormDataType) textPart = QHttpPart() textPart.setHeader(QNetworkRequest.ContentDispositionHeader, QVariant("form-data; name=\"driver\"")) textPart.setBody(QByteArray().append('Local')) filePart = QHttpPart() filePart.setHeader( QNetworkRequest.ContentDispositionHeader, QVariant("form-data; name=\"file\"; filename=\"{}\"".format( file_name))) file = QFile(data_url) file.open(QIODevice.ReadOnly) filePart.setBodyDevice(file) file.setParent( multiPart ) # we cannot delete the file now, so delete it with the multiPart multiPart.append(filePart) multiPart.append(textPart) service_url = '/'.join([ QSettings().value( 'Asistente-LADM_COL/sources/service_endpoint', DEFAULT_ENDPOINT_SOURCE_SERVICE), SOURCE_SERVICE_UPLOAD_SUFFIX ]) request = QNetworkRequest(QUrl(service_url)) reply = nam.post(request, multiPart) #reply.uploadProgress.connect(upload_dialog.update_current_progress) reply.error.connect(self.error_returned) multiPart.setParent(reply) # We'll block execution until we get response from the server loop = QEventLoop() reply.finished.connect(loop.quit) loop.exec_() response = reply.readAll() data = QTextStream(response, QIODevice.ReadOnly) content = data.readAll() if content is None: self.log.logMessage( "There was an error uploading file '{}'".format(data_url), PLUGIN_NAME, Qgis.Critical) upload_errors += 1 continue try: response = json.loads(content) except json.decoder.JSONDecodeError: self.log.logMessage( "Couldn't parse JSON response from server for file '{}'!!!" .format(data_url), PLUGIN_NAME, Qgis.Critical) upload_errors += 1 continue if 'error' in response: self.log.logMessage( "STATUS: {}. ERROR: {} MESSAGE: {} FILE: {}".format( response['status'], response['error'], response['message'], data_url), PLUGIN_NAME, Qgis.Critical) upload_errors += 1 continue reply.deleteLater() if 'url' not in response: self.log.logMessage( "'url' attribute not found in JSON response for file '{}'!" .format(data_url), PLUGIN_NAME, Qgis.Critical) upload_errors += 1 continue url = self.get_file_url(response['url']) new_values[feature.id()] = {field_index: url} count += 1 upload_dialog.update_total_progress(count) if not_found > 0: self.message_with_duration_emitted.emit( QCoreApplication.translate( "SourceHandler", "{} out of {} records {} not uploaded to the document repository because {} file path is NULL or it couldn't be found in the local disk!" ).format( not_found, total, QCoreApplication.translate("SourceHandler", "was") if not_found == 1 else QCoreApplication.translate( "SourceHandler", "were"), QCoreApplication.translate("SourceHandler", "its") if not_found == 1 else QCoreApplication.translate( "SourceHandler", "their")), Qgis.Info, 0) if len(new_values): self.message_with_duration_emitted.emit( QCoreApplication.translate( "SourceHandler", "{} out of {} files {} uploaded to the document repository and {} remote location stored in the database!" ).format( len(new_values), total, QCoreApplication.translate("SourceHandler", "was") if len(new_values) == 1 else QCoreApplication.translate( "SourceHandler", "were"), QCoreApplication.translate("SourceHandler", "its") if len(new_values) == 1 else QCoreApplication.translate( "SourceHandler", "their")), Qgis.Info, 0) if upload_errors: self.message_with_duration_emitted.emit( QCoreApplication.translate( "SourceHandler", "{} out of {} files could not be uploaded to the document repository because of upload errors! See log for details." ).format(upload_errors, total), Qgis.Info, 0) return new_values
class NetManager(QObject): def __init__(self, parent): super().__init__(parent) self.network = QNetworkAccessManager(self) ############# def _pre_send_request(self,conn_info, endpoint, kw_request=dict()): assert(isinstance(endpoint,str)) request = make_conn_request(conn_info, endpoint,**kw_request) return request def _post_send_request(self, reply, conn_info, kw_prop=dict()): set_qt_property(reply, conn_info=conn_info, **kw_prop) def _send_request(self,conn_info, endpoint, kw_request=dict(), kw_prop=dict()): request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.get(request) self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply ############# # TODO: remove callback params def get_statistics(self, conn_info): reply = self._get_space_(conn_info, "statistics") timeout = TIMEOUT_COUNT QTimer.singleShot(timeout, reply.abort) return reply def get_count(self, conn_info): reply = self._get_space_(conn_info, "count") return reply def get_meta(self, conn_info): return self._get_space_(conn_info, "space_meta") def _get_space_(self, conn_info, reply_tag): tag = "/" + reply_tag if reply_tag != "space_meta" else "" endpoint = "/spaces/{space_id}" + tag kw_request = dict() kw_prop = dict(reply_tag=reply_tag) return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def list_spaces(self, conn_info): endpoint = "/spaces" kw_request = dict(includeRights="true") kw_prop = dict(reply_tag="spaces") return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def add_space(self, conn_info, space_info): space_info = prepare_new_space_info(space_info) endpoint = "/spaces" kw_request = dict(req_type="json") kw_prop = dict(reply_tag="add_space") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.post(request, make_payload(space_info)) self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply def edit_space(self, conn_info, space_info): endpoint = "/spaces/{space_id}" kw_request = dict(req_type="json") kw_prop = dict(reply_tag="edit_space") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) buffer = make_buffer(space_info) reply = self.network.sendCustomRequest(request, b"PATCH", buffer) buffer.setParent(reply) self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply def del_space(self, conn_info): endpoint = "/spaces/{space_id}" kw_request = dict() kw_prop = dict(reply_tag="del_space") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"DELETE") self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply def load_features_bbox(self, conn_info, bbox, **kw): endpoint = "/spaces/{space_id}/bbox" kw_request = dict(bbox) kw_request.update(kw) kw_prop = dict(reply_tag="bbox") kw_prop.update(kw) kw_prop["bbox"] = bbox return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def load_features_tile(self, conn_info, tile_id="0", tile_schema="quadkey", **kw): reply_tag = "tile" tile_url = "tile/{tile_schema}/{tile_id}".format( tile_schema=tile_schema, tile_id=tile_id) endpoint = "/spaces/{space_id}/" + tile_url return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw) def load_features_iterate(self, conn_info, **kw_iterate): reply_tag = kw_iterate.pop("reply_tag","iterate") endpoint = "/spaces/{space_id}/iterate" return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate) def load_features_search(self, conn_info, **kw_iterate): reply_tag = kw_iterate.pop("reply_tag","search") endpoint = "/spaces/{space_id}/search" return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate) def _load_features_endpoint(self, endpoint, conn_info, reply_tag=None, **kw_iterate): """ Iterate through all ordered features (no feature is repeated twice) """ kw_request = dict(kw_iterate) kw_prop = dict(reply_tag=reply_tag) kw_prop.update(kw_iterate) return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) ###### feature function def add_features(self, conn_info, added_feat, **kw): send_request = self.network.post # create or modify (merge existing feature with payload) # might add attributes return self._add_features(conn_info, added_feat, send_request, **kw) def modify_features(self, conn_info, added_feat, **kw): return self.add_features(conn_info, added_feat, **kw) def replace_features(self, conn_info, added_feat, **kw): send_request = self.network.put # create or replace (replace existing feature with payload) # might add or drop attributes return self._add_features(conn_info, added_feat, send_request, **kw) def _add_features(self, conn_info, added_feat, send_request, **kw): # POST, payload: list of FeatureCollection endpoint = "/spaces/{space_id}/features" if "tags" in kw: kw["addTags"] = kw["tags"] kw_request = dict(req_type="geo", **kw) # kw: query kw_prop = dict(reply_tag="add_feat") kw_prop.update(kw) request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) payload = make_payload(added_feat) reply = send_request(request, payload) self._post_send_request(reply, conn_info, kw_prop=kw_prop) #parallel case (merge output ? split input?) return reply def del_features(self, conn_info, removed_feat, **kw): # DELETE by Query URL, required list of feat_id query_del = {"id": ",".join(str(i) for i in removed_feat)} kw.update(query_del) endpoint = "/spaces/{space_id}/features" kw_request = dict(kw) # kw: query kw_prop = dict(reply_tag="del_feat") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"DELETE") self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply
class NetManager(QObject): def __init__(self, parent): super().__init__(parent) self.network = QNetworkAccessManager(self) ############# def _pre_send_request(self,conn_info, endpoint, kw_request=dict()): assert(isinstance(endpoint,str)) request = make_conn_request(conn_info, endpoint,**kw_request) return request def _post_send_request(self, reply, conn_info, kw_prop=dict()): set_qt_property(reply, conn_info=conn_info, **kw_prop) def _send_request(self,conn_info, endpoint, kw_request=dict(), kw_prop=dict()): request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.get(request) self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply ############# # TODO: remove callback params def get_statistics(self, conn_info): return self._get_space_(conn_info, "statistics") def get_count(self, conn_info): reply = self._get_space_(conn_info, "count") # timeout = 1000 # QTimer.singleShot(timeout, reply.abort) return reply def get_meta(self, conn_info): return self._get_space_(conn_info, "space_meta") def _get_space_(self, conn_info, reply_tag): tag = "/" + reply_tag if reply_tag != "space_meta" else "" endpoint = "/spaces/{space_id}" + tag kw_request = dict() kw_prop = dict(reply_tag=reply_tag) return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def list_spaces(self, conn_info): endpoint = "/spaces" kw_request = dict(includeRights="true") kw_prop = dict(reply_tag="spaces") return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def add_space(self, conn_info, space_info): space_info = prepare_new_space_info(space_info) endpoint = "/spaces" kw_request = dict(req_type="json") kw_prop = dict(reply_tag="add_space") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.post(request, make_payload(space_info)) self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply def edit_space(self, conn_info, space_info): endpoint = "/spaces/{space_id}" kw_request = dict(req_type="json") kw_prop = dict(reply_tag="edit_space") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"PATCH", make_payload(space_info)) self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply def del_space(self, conn_info): endpoint = "/spaces/{space_id}" kw_request = dict() kw_prop = dict(reply_tag="del_space") request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"DELETE") self._post_send_request(reply,conn_info, kw_prop=kw_prop) return reply def load_features_bbox(self, conn_info, bbox, **kw): endpoint = "/spaces/{space_id}/bbox" kw_request = dict(bbox) kw_request.update(kw) kw_prop = dict(reply_tag="bbox") kw_prop.update(kw) kw_prop["bbox"] = bbox return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def load_features_iterate(self, conn_info, **kw_iterate): reply_tag = kw_iterate.pop("reply_tag","iterate") endpoint = "/spaces/{space_id}/iterate" return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate) def load_features_search(self, conn_info, **kw_iterate): reply_tag = kw_iterate.pop("reply_tag","search") endpoint = "/spaces/{space_id}/search" return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate) def _load_features_endpoint(self, endpoint, conn_info, reply_tag=None, **kw_iterate): """ Iterate through all ordered features (no feature is repeated twice) """ kw_request = dict(kw_iterate) kw_prop = dict(reply_tag=reply_tag) kw_prop.update(kw_iterate) return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) ###### feature function def add_features(self, conn_info, added_feat, layer_id=None, **kw): # POST, payload: list of FeatureCollection endpoint = "/spaces/{space_id}/features" kw_request = dict(req_type="geo", **kw) # kw: query kw_prop = dict(reply_tag="add_feat",layer_id=layer_id) kw_prop.update(kw) request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) buffer = make_payload(added_feat) reply = self.network.post(request, buffer) self._post_send_request(reply, conn_info, kw_prop=kw_prop) #parallel case (merge output ? split input?) return reply def del_features(self, conn_info, removed_feat, layer_id, **kw): # DELETE by Query URL, required list of feat_id query_del = {"id": ",".join(str(i) for i in removed_feat)} kw.update(query_del) endpoint = "/spaces/{space_id}/features" kw_request = dict(kw) # kw: query kw_prop = dict(reply_tag="del_feat",layer_id=layer_id) request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"DELETE") self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply def sync(self, conn_info, feat, layer_id, **kw): added_feat, removed_feat = feat token, space_id = conn_info.get_xyz_space() if not added_feat is None: self.add_features(conn_info, added_feat, layer_id) if len(removed_feat): self.del_features(conn_info, removed_feat, layer_id)
class KeyWordsSelection: def __init__(self, auth_file): logger.debug("\n================== ISOGEO API WITH QT ==================") # getting credentials : logger.debug("Getting credentials") self.utils = IsogeoUtils() self.app_creds = self.utils.credentials_loader(auth_file) self.app_id = self.app_creds.get("client_id") self.app_secrets = self.app_creds.get("client_secret") # for connection : self.naMngr = QNetworkAccessManager() self.token_url = "https://id.api.isogeo.com/oauth/token" self.request_url = ( "https://v1.api.isogeo.com/resources/search?_limit=0&_offset=0" ) # init variables : self.token = "" self.search_type = "init" self.checked_kw = {} # for ui (launch and display): logger.debug("Processing and displaying UI") self.app = QApplication(sys.argv) self.ui = AuthWidget() self.ui.resize(400, 100) self.pysdk_checking() self.api_authentification() self.ui.btn_reset.pressed.connect(self.reset) self.ui.show() self.app.exec() def pysdk_checking(self): logger.debug("\n--------------- isogeo-pysdk ---------------") logger.debug("Checking credentials") try: isogeo = Isogeo(self.app_id, self.app_secrets) isogeo.connect() except OSError as e: logger.debug("Credentials issue : {}".format(e)) return except ValueError as e: logger.debug("Credentials issue : {}".format(e)) return if self.search_type == "init": result = isogeo.search( whole_share=0, page_size=0, augment=0, tags_as_dicts=1 ) else: query = " ".join(list(self.checked_kw.keys())) result = isogeo.search( whole_share=0, page_size=0, augment=0, tags_as_dicts=1, query=query ) self.tags_expected = result.get("tags") self.kw_expected_nb = len(self.tags_expected.get("keywords")) self.ui.lbl_expected.setText( "Expected : {} resources and {} keywords".format( result.get("total"), self.kw_expected_nb ) ) logger.debug( "isogeo-pysdk validates the authentication file, {} accessible resources".format( result.get("total") ) ) def api_authentification(self): logger.debug("\n------------------ Authentication ------------------") # creating credentials header crd_header_value = QByteArray() crd_header_value.append("Basic ") crd_header_value.append( base64.b64encode("{}:{}".format(self.app_id, self.app_secrets).encode()) ) crd_header_name = QByteArray() crd_header_name.append("Authorization") # creating Content-Type header ct_header_value = QByteArray() ct_header_value.append("application/json") # creating request token_rqst = QNetworkRequest(QUrl(self.token_url)) # setting headers token_rqst.setRawHeader(crd_header_name, crd_header_value) token_rqst.setHeader(token_rqst.ContentTypeHeader, ct_header_value) # creating data data = QByteArray() data.append(urlencode({"grant_type": "client_credentials"})) # requesting and handle reply logger.debug("Asking for token") token_reply = self.naMngr.post(token_rqst, data) token_reply.finished.connect(partial(self.api_handle_token, reply=token_reply)) def api_handle_token(self, reply): logger.debug("Token asked and API reply received") # formating API response bytarray = reply.readAll() content = bytarray.data().decode("utf8") # check API response structure try: parsed_content = json.loads(content) except: logger.debug("Reply format issue") return # check API response content if "access_token" in parsed_content: self.token = "Bearer " + parsed_content.get("access_token") self.api_get_request() else: logger.debug("ya une couille dans la magouille : {}".format(parsed_content)) def api_get_request(self): logger.debug("\n----------------- Sending request -----------------") # creating credentials header crd_header_value = QByteArray() crd_header_value.append(self.token) crd_header_name = QByteArray() crd_header_name.append("Authorization") # creating request rqst = QNetworkRequest(QUrl(self.request_url)) # setting credentials header rqst.setRawHeader(crd_header_name, crd_header_value) # sending request rqst_reply = self.naMngr.get(rqst) rqst_reply.finished.connect(partial(self.api_handle_request, reply=rqst_reply)) def api_handle_request(self, reply): logger.debug("Request sent and API reply received") bytarray = reply.readAll() content = bytarray.data().decode("utf8") if reply.error() == 0: # check API response structure try: parsed_content = json.loads(content) except: logger.debug("Reply format issue") return # check API response content self.tags_found = parsed_content.get("tags") self.kw_found = {} for tag in sorted(self.tags_found): if tag.startswith("keyword:is"): self.kw_found[tag] = self.tags_found.get(tag) else: pass # displaying result self.ui.lbl_found.setText( "Found : {} resources and {} keywords".format( parsed_content.get("total"), len(self.kw_found) ) ) if self.search_type == "init": if len(self.kw_found) == self.kw_expected_nb: logger.debug("!!! It's working !!!") else: logger.debug("It's NOT working") else: pass # filling keywords checkable combo box self.pop_kw_cbbox() elif self.search_type != "init": logger.debug("token expired, renewing it") self.api_authentification() else: pass def pop_kw_cbbox(self): logger.debug("\n-------------- Poping Keywords ComboBox --------------") # to prepare the filling self.ui.kw_cbbox.clear() if self.search_type != "init": self.ui.kw_cbbox.activated.disconnect(self.get_checked_kw) # filling the combobox with checkable items self.ui.kw_cbbox.addItem("-- Keywords --") first_item = self.ui.kw_cbbox.model().item(0, 0) first_item.setEnabled(False) i = 1 for kw_code, kw_lbl in self.kw_found.items(): if self.search_type == "kw" and kw_code in self.checked_kw.keys(): self.ui.kw_cbbox.insertItem(1, kw_lbl) item = self.ui.kw_cbbox.model().item(1, 0) item.setCheckState(Qt.Checked) else: self.ui.kw_cbbox.addItem(kw_lbl) item = self.ui.kw_cbbox.model().item(i, 0) item.setCheckState(Qt.Unchecked) item.setData(kw_code, 32) i += 1 logger.debug("Keywords Combobox filled") self.ui.kw_cbbox.setEnabled(True) # connecting to a signal returning the checked item's index self.ui.kw_cbbox.activated.connect(self.get_checked_kw) def get_checked_kw(self, index): logger.debug("\n------------ Getting Checked Keywords ------------") self.ui.kw_cbbox.setEnabled(False) self.ui.kw_cbbox.setCurrentText(self.ui.kw_cbbox.itemText(index)) # Testing if checked keyword is already in the dict is easier than # testing if the user checked or unchecked it : # removing the selected keyword from the dict if it is already in if self.ui.kw_cbbox.itemData(index, 32) in self.checked_kw.keys(): del self.checked_kw[self.ui.kw_cbbox.itemData(index, 32)] # adding the selected keyword to the dict if it is not already in else: self.checked_kw[ self.ui.kw_cbbox.itemData(index, 32) ] = self.ui.kw_cbbox.itemText(index) logger.debug("ckeched kw : {}".format(self.checked_kw)) self.ui.lbl_selection.setText( "{} keywords selected".format(len(self.checked_kw)) ) self.ui.kw_cbbox.setToolTip(" / ".join(list(self.checked_kw.values()))) # now selected keywords are stocked, time to request the API self.kw_search() def kw_search(self): logger.debug("\n------------ Searching with keywords -------------") # preparing the request self.search_type = "kw" self.request_url = self.url_builder() self.pysdk_checking() # launching the request self.api_get_request() def url_builder(self): logger.debug("\n------------------ Building URL ------------------") # adding selected keywords to the request URL search_url = "https://v1.api.isogeo.com/resources/search?q=" for kw in self.checked_kw: search_url += "{} ".format(str(kw)) logger.debug("URL : {}".format(search_url)) return search_url[:-1] def reset(self): logger.debug("----------------- RESET -------------------") self.search_type = "reset" self.checked_kw = {} self.request_url = ( "https://v1.api.isogeo.com/resources/search?_limit=0&_offset=0" ) self.ui.lbl_selection.setText("") self.ui.kw_cbbox.setToolTip("") self.pysdk_checking() self.api_get_request()
class ApiConnection: def __init__(self, auth_file): logger.debug( "\n================== ISOGEO API WITH QT ==================") # getting credentials : logger.debug("Getting credentials") self.utils = IsogeoUtils() self.app_creds = self.utils.credentials_loader(auth_file) self.app_id = self.app_creds.get("client_id") self.app_secrets = self.app_creds.get("client_secret") # for connection : self.naMngr = QNetworkAccessManager() self.token_url = "https://id.api.isogeo.com/oauth/token" self.request_url = ( "https://v1.api.isogeo.com/resources/search?_limit=0&_offset=0") self.token = "" # for ui : logger.debug("Processing and displaying UI") self.app = QApplication([]) self.ui = AuthWidget() self.ui.resize(350, 100) self.ui.btn.clicked.connect(self.api_authentification) self.pysdk_checking() self.ui.show() self.app.exec() def pysdk_checking(self): logger.debug("\n------------------ isogeo-pysdk ------------------") logger.debug("Checking credentials") try: isogeo = Isogeo(self.app_id, self.app_secrets) isogeo.connect() except OSError as e: logger.debug("Credentials issue : {}".format(e)) return except ValueError as e: logger.debug("Credentials issue : {}".format(e)) return self.md_expected = isogeo.search(whole_share=0, page_size=0, augment=0).get("total") self.ui.lbl_expected.setText("{} expected resources".format( self.md_expected)) logger.debug( "isogeo-pysdk validates the authentication file, {} accessible resources" .format(self.md_expected)) def api_authentification(self): logger.debug("\n------------------ Authentication ------------------") # creating credentials header logger.debug("Creating credentials header") crd_header_value = QByteArray() crd_header_value.append("Basic ") crd_header_value.append( base64.b64encode("{}:{}".format(self.app_id, self.app_secrets).encode())) crd_header_name = QByteArray() crd_header_name.append("Authorization") # creating Content-Type header logger.debug("Creating 'Content-Type' header") ct_header_value = QByteArray() ct_header_value.append("application/json") # creating request token_rqst = QNetworkRequest(QUrl(self.token_url)) logger.debug("Creating token request : {}".format(token_rqst.url())) # setting headers token_rqst.setRawHeader(crd_header_name, crd_header_value) logger.debug("Setting credentials header : {}".format( token_rqst.rawHeader(crd_header_name))) token_rqst.setHeader(token_rqst.ContentTypeHeader, ct_header_value) logger.debug("Setting 'Content-Type' header : {}".format( token_rqst.header(token_rqst.ContentTypeHeader))) # creating data data = QByteArray() data.append(urlencode({"grant_type": "client_credentials"})) logger.debug("Creating data : {}".format(data)) # requesting and handle reply logger.debug("Asking for token") token_reply = self.naMngr.post(token_rqst, data) token_reply.finished.connect( partial(self.api_handle_token, reply=token_reply)) def api_handle_token(self, reply): logger.debug("\n------------------ Token retrieval ------------------") logger.debug("Token asked and API reply received : {}".format(reply)) logger.debug("Storage and formatting of the reply") bytarray = reply.readAll() content = bytarray.data().decode("utf8") # check API response structure try: parsed_content = json.loads(content) except: logger.debug("Reply format issue") return logger.debug("Reply format is good") if "access_token" in parsed_content: self.token = "Bearer " + parsed_content.get("access_token") logger.debug("TOKEN STORED") self.api_get_request() else: logger.debug( "ya une couille dans la magouille : {}".format(parsed_content)) def api_get_request(self): logger.debug("\n------------------ Sending request ------------------") # creating credentials header logger.debug("Creating credentials header") crd_header_value = QByteArray() crd_header_value.append(self.token) crd_header_name = QByteArray() crd_header_name.append("Authorization") # creating request rqst = QNetworkRequest(QUrl(self.request_url)) logger.debug("Creating request : {}".format(rqst.url())) # setting credentials header rqst.setRawHeader(crd_header_name, crd_header_value) logger.debug("Setting credentials header : {}".format( rqst.rawHeader(crd_header_name))) # sending request rqst_reply = self.naMngr.get(rqst) logger.debug("Sending request") rqst_reply.finished.connect( partial(self.api_handle_request, reply=rqst_reply)) def api_handle_request(self, reply): logger.debug("\n------------------ Reply retrieval ------------------") logger.debug("Request sent and API reply received : {}".format(reply)) logger.debug("Storage and formatting of the reply") bytarray = reply.readAll() content = bytarray.data().decode("utf8") # check API response structure try: parsed_content = json.loads(content) except: logger.debug("Reply format issue") return logger.debug("Reply format is good") self.md_found = parsed_content.get("total") logger.debug("RESULT STORED") self.ui.lbl_found.setText("{} resources found".format(self.md_found)) if self.md_found == self.md_expected: logger.debug("!!! It's working !!!") else: logger.debug("It's NOT working")
class NetManager(QObject): def __init__(self, parent): super().__init__(parent) self.network = QNetworkAccessManager(self) ############# def _pre_send_request(self, conn_info, endpoint, kw_request=dict()): assert (isinstance(endpoint, str)) request = make_conn_request(conn_info, endpoint, **kw_request) return request def _post_send_request(self, reply, conn_info, kw_prop=dict()): set_qt_property(reply, conn_info=conn_info, **kw_prop) def _send_request(self, conn_info, endpoint, kw_request=dict(), kw_prop=dict()): request = self._pre_send_request(conn_info, endpoint, kw_request=kw_request) reply = self.network.get(request) self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply ############# # TODO: remove callback params def get_statistics(self, conn_info): reply = self._get_space_(conn_info, "statistics") timeout = TIMEOUT_COUNT QTimer.singleShot(timeout, reply.abort) return reply def get_count(self, conn_info): reply = self._get_space_(conn_info, "count") return reply def get_meta(self, conn_info): return self._get_space_(conn_info, "space_meta") def _get_space_(self, conn_info, reply_tag): tag = "/" + reply_tag if reply_tag != "space_meta" else "" endpoint = "/spaces/{space_id}" + tag kw_request = dict() kw_prop = dict(reply_tag=reply_tag) return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def list_spaces(self, conn_info): endpoint = "/spaces" kw_request = dict(includeRights="true") kw_prop = dict(reply_tag="spaces") return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def add_space(self, conn_info, space_info): space_info = prepare_new_space_info(space_info) endpoint = "/spaces" kw_request = dict(req_type="json") kw_prop = dict(reply_tag="add_space") request = self._pre_send_request(conn_info, endpoint, kw_request=kw_request) reply = self.network.post(request, make_payload(space_info)) self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply def edit_space(self, conn_info, space_info): endpoint = "/spaces/{space_id}" kw_request = dict(req_type="json") kw_prop = dict(reply_tag="edit_space") request = self._pre_send_request(conn_info, endpoint, kw_request=kw_request) buffer = make_buffer(space_info) reply = self.network.sendCustomRequest(request, b"PATCH", buffer) buffer.setParent(reply) self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply def del_space(self, conn_info): endpoint = "/spaces/{space_id}" kw_request = dict() kw_prop = dict(reply_tag="del_space") request = self._pre_send_request(conn_info, endpoint, kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"DELETE") self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply def _prefix_query(self, txt, prefix="p.", prefixes=tuple()): """ return prefix to xyz query: prefixes = ["p.", "f."] # add prefix if none in prefixes exists prefixes = [] # always add prefix, dont check existing """ return prefix if not any(map(txt.startswith, prefixes)) else "" def _process_queries(self, kw): selection = ",".join( "{prefix}{name}".format(name=p, prefix=self._prefix_query(p, "p.")) for p in kw.pop("selection", "").split(",") if p) if selection: kw["selection"] = selection self._process_raw_queries(kw) def _process_raw_queries(self, kw): filters = [ "{prefix}{name}{operator}{value}".format(name=p["name"], operator=p["operator"], value=p["values"], prefix=self._prefix_query( p["name"], "p.")) for p in kw.pop("filters", list()) ] if filters: kw.setdefault("raw_queries", list()).extend(filters) def load_features_bbox(self, conn_info, bbox, **kw): reply_tag = "bbox" endpoint = "/spaces/{space_id}/bbox" self._process_queries(kw) kw_request = dict(bbox, **kw) kw_prop = dict(reply_tag=reply_tag, bbox=bbox, **kw) return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop) def load_features_tile(self, conn_info, tile_id="0", tile_schema="quadkey", **kw): reply_tag = "tile" kw_tile = dict(tile_schema=tile_schema, tile_id=tile_id) tile_url = "tile/{tile_schema}/{tile_id}".format(**kw_tile) endpoint = "/spaces/{space_id}/" + tile_url self._process_queries(kw) kw_prop = dict(reply_tag=reply_tag, **dict(kw, **kw_tile)) return self._send_request(conn_info, endpoint, kw_request=kw, kw_prop=kw_prop) def load_features_iterate(self, conn_info, **kw): reply_tag = kw.pop("reply_tag", "iterate") endpoint = "/spaces/{space_id}/iterate" self._process_queries(kw) kw_prop = dict(reply_tag=reply_tag, **kw) return self._send_request(conn_info, endpoint, kw_request=kw, kw_prop=kw_prop) def load_features_search(self, conn_info, **kw): reply_tag = kw.pop("reply_tag", "search") endpoint = "/spaces/{space_id}/search" self._process_queries(kw) kw_prop = dict(reply_tag=reply_tag, **kw) return self._send_request(conn_info, endpoint, kw_request=kw, kw_prop=kw_prop) ###### feature function def add_features(self, conn_info, added_feat, **kw): send_request = self.network.post # create or modify (merge existing feature with payload) # might add attributes return self._add_features(conn_info, added_feat, send_request, **kw) def modify_features(self, conn_info, added_feat, **kw): return self.add_features(conn_info, added_feat, **kw) def replace_features(self, conn_info, added_feat, **kw): send_request = self.network.put # create or replace (replace existing feature with payload) # might add or drop attributes return self._add_features(conn_info, added_feat, send_request, **kw) def _add_features(self, conn_info, added_feat, send_request, **kw): # POST, payload: list of FeatureCollection endpoint = "/spaces/{space_id}/features" if "tags" in kw: kw["addTags"] = kw["tags"] kw_request = dict(req_type="geo", **kw) # kw: query kw_prop = dict(reply_tag="add_feat") kw_prop.update(kw) request = self._pre_send_request(conn_info, endpoint, kw_request=kw_request) payload = make_payload(added_feat) reply = send_request(request, payload) self._post_send_request(reply, conn_info, kw_prop=kw_prop) #parallel case (merge output ? split input?) return reply def del_features(self, conn_info, removed_feat, **kw): # DELETE by Query URL, required list of feat_id query_del = {"id": ",".join(str(i) for i in removed_feat)} kw.update(query_del) endpoint = "/spaces/{space_id}/features" kw_request = dict(kw) # kw: query kw_prop = dict(reply_tag="del_feat") request = self._pre_send_request(conn_info, endpoint, kw_request=kw_request) reply = self.network.sendCustomRequest(request, b"DELETE") self._post_send_request(reply, conn_info, kw_prop=kw_prop) return reply
class AccessSite(QObject): abortReply = pyqtSignal() finished = pyqtSignal(dict) send_data = pyqtSignal(QByteArray) status_download = pyqtSignal(int, int) status_erros = pyqtSignal(list) ErrorCodeAttribute = { 10: 'Canceled request', 400: 'Bad request syntax', 401: 'Unauthorized', 402: 'Payment required', 403: 'Forbidden', 404: 'Not found', 500: 'Internal error', 501: 'Not implemented', 502: 'Bad Gateway' } def __init__(self): super().__init__() self.isKill = None self.responseAllFinished = None self.nam = QNetworkAccessManager(self) self.nam.finished.connect( self.replyFinished ) def _connectReply(self, reply, isConnect=True): ss = [ { 'signal': reply.readyRead, 'slot': self.readyRead }, { 'signal': reply.sslErrors, 'slot': self.sslErrors } ] if isConnect: if not self.responseAllFinished: reply.downloadProgress.connect( self.downloadProgress ) for item in ss: item['signal'].connect( item['slot'] ) else: if not self.responseAllFinished: reply.downloadProgress.disconnect( self.downloadProgress ) for item in ss: item['signal'].disconnect( item['slot'] ) def _closeReply(self, reply): def connect(isConnect=True): f_nam = self.nam.finished.connect if isConnect else self.nam.finished.disconnect f_nam( self.replyFinished ) if not self.responseAllFinished: self._connectReply( reply, isConnect ) # reply.close() call replyFinished connect(False) reply.close() connect() reply.deleteLater() def _redirectionReply(self, reply, url): self._closeReply( reply ) if url.isRelative(): url = url.resolved( url ) request = QNetworkRequest( url ) reply = self.nam.get( request ) if reply is None: response = { 'isOk': False, 'message': "Netwok error", 'errorCode': -1 } self.finished.emit( response ) return if not self.responseAllFinished: self._connectReply( reply ) def _emitErrorCodeAttribute(self, code, reply): msg = 'Error network' if not code in self.ErrorCodeAttribute.keys() else self.ErrorCodeAttribute[ code ] response = { 'isOk': False, 'message': msg, 'errorCode': code } self._closeReply( reply ) self.finished.emit( response ) def _checkRedirectionAttribute(self, reply): urlRedir = reply.attribute( QNetworkRequest.RedirectionTargetAttribute ) if not urlRedir is None and urlRedir != reply.url(): self._redirectionReply( reply, urlRedir ) return { 'isOk': True } codeAttribute = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute ) if not ( 200 <= codeAttribute <= 299 ): self._emitErrorCodeAttribute( codeAttribute, reply ) return { 'isOk': False } return { 'isOk': True } def _clearResponse(self, response): if 'data' in response: response['data'].clear() del response[ 'data' ] if 'statusRequest' in response: del response['statusRequest'] if response['isOk']: if 'errorCode' in response: del response['errorCode'] def requestUrl(self, paramsAccess, addFinishedResponse, setFinished): @pyqtSlot(dict) def finished( response): loop.quit() self.finished.disconnect( finished ) if 'notResponseAllFinished' in paramsAccess: self.send_data.disconnect( paramsAccess['notResponseAllFinished']['writePackageImage'] ) self.status_download.disconnect( paramsAccess['notResponseAllFinished']['progressPackageImage'] ) response = addFinishedResponse( response ) if response['isOk']: self._clearResponse( response ) setFinished( response ) def run(): def sslVerifyNone(req): conf = req.sslConfiguration() conf.setPeerVerifyMode( QSslSocket.VerifyNone ) req.setSslConfiguration( conf ) req = QNetworkRequest( url ) sslVerifyNone( req ) # Need for Windows, error 'Handshake failed' if json_request is None: reply = self.nam.get( req ) else: req.setHeader( QNetworkRequest.ContentTypeHeader, "application/json" ) data = QByteArray() data.append( json.dumps( json_request ) ) reply = self.nam.post( req, data ) if reply is None: response = { 'isOk': False, 'message': "Network error", 'errorCode': -1 } self.finished.emit( response ) return self.abortReply.connect( reply.abort ) if not self.responseAllFinished: self._connectReply( reply ) loop = QEventLoop() self.finished.connect( finished ) json_request = None if not 'json_request' in paramsAccess else paramsAccess['json_request'] self.responseAllFinished = True if 'notResponseAllFinished' in paramsAccess: self.responseAllFinished = False self.send_data.connect( paramsAccess['notResponseAllFinished']['writePackageImage'] ) self.status_download.connect( paramsAccess['notResponseAllFinished']['progressPackageImage'] ) url = paramsAccess['url'] if 'credential' in paramsAccess: userInfo = "{user}:{pwd}".format( user=paramsAccess['credential']['user'], pwd=paramsAccess['credential']['password'] ) url.setUserInfo( userInfo ) self.isKill = False run() loop.exec_() def isHostLive(self, url, setFinished): def addFinishedResponse(response): if response['isOk']: return response else: if response['errorCode'] == QNetworkReply.HostNotFoundError: response['message'] = "{}\nURL = {}".format( response['message'], self.urlGeoserver ) else: response['isOk'] = True return response p = { 'url': QUrl( url ) } self.requestUrl( p, addFinishedResponse, setFinished ) def getThumbnail(self, url, setFinished): def addFinishedResponse(response): if not response['isOk']: return response if 'data' in response: # The user can quickly change a item pixmap = QPixmap() if not pixmap.loadFromData( response['data'] ): response['isOk'] = False response['message'] = 'Invalid image from Mapbiomas server' else: response['thumbnail'] = pixmap return response p = { 'url': QUrl( url ) } self.requestUrl( p, addFinishedResponse, setFinished ) @pyqtSlot('QNetworkReply*') def replyFinished(self, reply) : if self.isKill: self._emitErrorCodeAttribute(10, reply ) return if reply.error() != QNetworkReply.NoError: response = { 'isOk': False, 'message': reply.errorString(), 'errorCode': reply.error() } self._closeReply( reply ) self.finished.emit( response ) return r = self._checkRedirectionAttribute( reply ) if not r['isOk']: return statusRequest = { 'contentTypeHeader': reply.header( QNetworkRequest.ContentTypeHeader ), 'lastModifiedHeader': reply.header( QNetworkRequest.LastModifiedHeader ), 'contentLengthHeader': reply.header( QNetworkRequest.ContentLengthHeader ), 'statusCodeAttribute': reply.attribute( QNetworkRequest.HttpStatusCodeAttribute ), 'reasonPhraseAttribute': reply.attribute( QNetworkRequest.HttpReasonPhraseAttribute ) } response = { 'isOk': True, 'statusRequest': statusRequest } if self.responseAllFinished: response['data'] = reply.readAll() self._closeReply( reply ) self.finished.emit( response ) @pyqtSlot() def readyRead(self): reply = self.sender() if self.isKill: self._emitErrorCodeAttribute(10, reply ) return r = self._checkRedirectionAttribute( reply ) if not r['isOk']: return if not reply.isOpen(): reply.open( QNetworkReply.ReadOnly ) data = reply.readAll() if data is None: return self.send_data.emit( data ) @pyqtSlot('qint64', 'qint64') def downloadProgress(self, bytesReceived, bytesTotal): reply = self.sender() if self.isKill: self._emitErrorCodeAttribute(10, reply ) else: self.status_download.emit( bytesReceived, bytesTotal ) @pyqtSlot('QList<QSslError>') def sslErrors(self, errors): reply = self.sender() lstErros = map( lambda e: e.errorString(), errors ) self.status_erros.emit( lstErros ) reply.ignoreSslErrors() @staticmethod def loadJsonData(response): data = response['data'].data() sdata = str(data, encoding='utf-8') return json.loads( sdata )