def upload_files(self, layer, field_index, features):
"""
Upload given features' source files to remote server and return a dict
formatted as changeAttributeValues expects to update 'datos' attribute
to a remote location.
"""
if not QSettings().value(
'Asistente-LADM_COL/sources/document_repository', False, bool):
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"The source files were not uploaded to the document repository because you have that option unchecked. You can still upload the source files later using the 'Upload Pending Source Files' menu."
), Qgis.Info, 10)
return dict()
# Test if we have Internet connection and a valid service
dlg = self.qgis_utils.get_settings_dialog()
res, msg = dlg.is_source_service_valid()
if not res:
msg['text'] = QCoreApplication.translate(
"SourceHandler",
"No file could be uploaded to the document repository. You can do it later from the 'Upload Pending Source Files' menu. Reason: {}"
).format(msg['text'])
self.message_with_duration_emitted.emit(
msg['text'], Qgis.Info,
20) # The data is still saved, so always show Info msg
return dict()
file_features = [
feature for feature in features if not feature[field_index] == NULL
and os.path.isfile(feature[field_index])
]
total = len(features)
not_found = total - len(file_features)
upload_dialog = UploadProgressDialog(len(file_features), not_found)
upload_dialog.show()
count = 0
upload_errors = 0
new_values = dict()
for feature in file_features:
data_url = feature[field_index]
file_name = os.path.basename(data_url)
nam = QNetworkAccessManager()
#reply.downloadProgress.connect(upload_dialog.update_current_progress)
multiPart = QHttpMultiPart(QHttpMultiPart.FormDataType)
textPart = QHttpPart()
textPart.setHeader(QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"driver\""))
textPart.setBody(QByteArray().append('Local'))
filePart = QHttpPart()
filePart.setHeader(
QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"file\"; filename=\"{}\"".format(
file_name)))
file = QFile(data_url)
file.open(QIODevice.ReadOnly)
filePart.setBodyDevice(file)
file.setParent(
multiPart
) # we cannot delete the file now, so delete it with the multiPart
multiPart.append(filePart)
multiPart.append(textPart)
service_url = '/'.join([
QSettings().value(
'Asistente-LADM_COL/sources/service_endpoint',
DEFAULT_ENDPOINT_SOURCE_SERVICE),
SOURCE_SERVICE_UPLOAD_SUFFIX
])
request = QNetworkRequest(QUrl(service_url))
reply = nam.post(request, multiPart)
#reply.uploadProgress.connect(upload_dialog.update_current_progress)
reply.error.connect(self.error_returned)
multiPart.setParent(reply)
# We'll block execution until we get response from the server
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
response = reply.readAll()
data = QTextStream(response, QIODevice.ReadOnly)
content = data.readAll()
if content is None:
self.log.logMessage(
"There was an error uploading file '{}'".format(data_url),
PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
try:
response = json.loads(content)
except json.decoder.JSONDecodeError:
self.log.logMessage(
"Couldn't parse JSON response from server for file '{}'!!!"
.format(data_url), PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
if 'error' in response:
self.log.logMessage(
"STATUS: {}. ERROR: {} MESSAGE: {} FILE: {}".format(
response['status'], response['error'],
response['message'], data_url), PLUGIN_NAME,
Qgis.Critical)
upload_errors += 1
continue
reply.deleteLater()
if 'url' not in response:
self.log.logMessage(
"'url' attribute not found in JSON response for file '{}'!"
.format(data_url), PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
url = self.get_file_url(response['url'])
new_values[feature.id()] = {field_index: url}
count += 1
upload_dialog.update_total_progress(count)
if not_found > 0:
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} records {} not uploaded to the document repository because {} file path is NULL or it couldn't be found in the local disk!"
).format(
not_found, total,
QCoreApplication.translate("SourceHandler", "was")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "their")), Qgis.Info, 0)
if len(new_values):
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files {} uploaded to the document repository and {} remote location stored in the database!"
).format(
len(new_values), total,
QCoreApplication.translate("SourceHandler", "was")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "their")), Qgis.Info, 0)
if upload_errors:
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files could not be uploaded to the document repository because of upload errors! See log for details."
).format(upload_errors, total), Qgis.Info, 0)
return new_values
class NetManager(QObject):
def __init__(self, parent):
super().__init__(parent)
self.network = QNetworkAccessManager(self)
#############
def _pre_send_request(self,conn_info, endpoint, kw_request=dict()):
assert(isinstance(endpoint,str))
request = make_conn_request(conn_info, endpoint,**kw_request)
return request
def _post_send_request(self, reply, conn_info, kw_prop=dict()):
set_qt_property(reply, conn_info=conn_info, **kw_prop)
def _send_request(self,conn_info, endpoint, kw_request=dict(), kw_prop=dict()):
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.get(request)
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
#############
# TODO: remove callback params
def get_statistics(self, conn_info):
reply = self._get_space_(conn_info, "statistics")
timeout = TIMEOUT_COUNT
QTimer.singleShot(timeout, reply.abort)
return reply
def get_count(self, conn_info):
reply = self._get_space_(conn_info, "count")
return reply
def get_meta(self, conn_info):
return self._get_space_(conn_info, "space_meta")
def _get_space_(self, conn_info, reply_tag):
tag = "/" + reply_tag if reply_tag != "space_meta" else ""
endpoint = "/spaces/{space_id}" + tag
kw_request = dict()
kw_prop = dict(reply_tag=reply_tag)
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
def list_spaces(self, conn_info):
endpoint = "/spaces"
kw_request = dict(includeRights="true")
kw_prop = dict(reply_tag="spaces")
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
def add_space(self, conn_info, space_info):
space_info = prepare_new_space_info(space_info)
endpoint = "/spaces"
kw_request = dict(req_type="json")
kw_prop = dict(reply_tag="add_space")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.post(request, make_payload(space_info))
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
def edit_space(self, conn_info, space_info):
endpoint = "/spaces/{space_id}"
kw_request = dict(req_type="json")
kw_prop = dict(reply_tag="edit_space")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
buffer = make_buffer(space_info)
reply = self.network.sendCustomRequest(request, b"PATCH", buffer)
buffer.setParent(reply)
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
def del_space(self, conn_info):
endpoint = "/spaces/{space_id}"
kw_request = dict()
kw_prop = dict(reply_tag="del_space")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"DELETE")
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
def load_features_bbox(self, conn_info, bbox, **kw):
endpoint = "/spaces/{space_id}/bbox"
kw_request = dict(bbox)
kw_request.update(kw)
kw_prop = dict(reply_tag="bbox")
kw_prop.update(kw)
kw_prop["bbox"] = bbox
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
def load_features_tile(self, conn_info, tile_id="0", tile_schema="quadkey", **kw):
reply_tag = "tile"
tile_url = "tile/{tile_schema}/{tile_id}".format(
tile_schema=tile_schema, tile_id=tile_id)
endpoint = "/spaces/{space_id}/" + tile_url
return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw)
def load_features_iterate(self, conn_info, **kw_iterate):
reply_tag = kw_iterate.pop("reply_tag","iterate")
endpoint = "/spaces/{space_id}/iterate"
return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate)
def load_features_search(self, conn_info, **kw_iterate):
reply_tag = kw_iterate.pop("reply_tag","search")
endpoint = "/spaces/{space_id}/search"
return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate)
def _load_features_endpoint(self, endpoint, conn_info, reply_tag=None, **kw_iterate):
""" Iterate through all ordered features (no feature is repeated twice)
"""
kw_request = dict(kw_iterate)
kw_prop = dict(reply_tag=reply_tag)
kw_prop.update(kw_iterate)
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
###### feature function
def add_features(self, conn_info, added_feat, **kw):
send_request = self.network.post # create or modify (merge existing feature with payload) # might add attributes
return self._add_features(conn_info, added_feat, send_request, **kw)
def modify_features(self, conn_info, added_feat, **kw):
return self.add_features(conn_info, added_feat, **kw)
def replace_features(self, conn_info, added_feat, **kw):
send_request = self.network.put # create or replace (replace existing feature with payload) # might add or drop attributes
return self._add_features(conn_info, added_feat, send_request, **kw)
def _add_features(self, conn_info, added_feat, send_request, **kw):
# POST, payload: list of FeatureCollection
endpoint = "/spaces/{space_id}/features"
if "tags" in kw: kw["addTags"] = kw["tags"]
kw_request = dict(req_type="geo", **kw) # kw: query
kw_prop = dict(reply_tag="add_feat")
kw_prop.update(kw)
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
payload = make_payload(added_feat)
reply = send_request(request, payload)
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
#parallel case (merge output ? split input?)
return reply
def del_features(self, conn_info, removed_feat, **kw):
# DELETE by Query URL, required list of feat_id
query_del = {"id": ",".join(str(i) for i in removed_feat)}
kw.update(query_del)
endpoint = "/spaces/{space_id}/features"
kw_request = dict(kw) # kw: query
kw_prop = dict(reply_tag="del_feat")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"DELETE")
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
class NetManager(QObject):
def __init__(self, parent):
super().__init__(parent)
self.network = QNetworkAccessManager(self)
#############
def _pre_send_request(self,conn_info, endpoint, kw_request=dict()):
assert(isinstance(endpoint,str))
request = make_conn_request(conn_info, endpoint,**kw_request)
return request
def _post_send_request(self, reply, conn_info, kw_prop=dict()):
set_qt_property(reply, conn_info=conn_info, **kw_prop)
def _send_request(self,conn_info, endpoint, kw_request=dict(), kw_prop=dict()):
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.get(request)
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
#############
# TODO: remove callback params
def get_statistics(self, conn_info):
return self._get_space_(conn_info, "statistics")
def get_count(self, conn_info):
reply = self._get_space_(conn_info, "count")
# timeout = 1000
# QTimer.singleShot(timeout, reply.abort)
return reply
def get_meta(self, conn_info):
return self._get_space_(conn_info, "space_meta")
def _get_space_(self, conn_info, reply_tag):
tag = "/" + reply_tag if reply_tag != "space_meta" else ""
endpoint = "/spaces/{space_id}" + tag
kw_request = dict()
kw_prop = dict(reply_tag=reply_tag)
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
def list_spaces(self, conn_info):
endpoint = "/spaces"
kw_request = dict(includeRights="true")
kw_prop = dict(reply_tag="spaces")
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
def add_space(self, conn_info, space_info):
space_info = prepare_new_space_info(space_info)
endpoint = "/spaces"
kw_request = dict(req_type="json")
kw_prop = dict(reply_tag="add_space")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.post(request, make_payload(space_info))
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
def edit_space(self, conn_info, space_info):
endpoint = "/spaces/{space_id}"
kw_request = dict(req_type="json")
kw_prop = dict(reply_tag="edit_space")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"PATCH", make_payload(space_info))
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
def del_space(self, conn_info):
endpoint = "/spaces/{space_id}"
kw_request = dict()
kw_prop = dict(reply_tag="del_space")
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"DELETE")
self._post_send_request(reply,conn_info, kw_prop=kw_prop)
return reply
def load_features_bbox(self, conn_info, bbox, **kw):
endpoint = "/spaces/{space_id}/bbox"
kw_request = dict(bbox)
kw_request.update(kw)
kw_prop = dict(reply_tag="bbox")
kw_prop.update(kw)
kw_prop["bbox"] = bbox
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
def load_features_iterate(self, conn_info, **kw_iterate):
reply_tag = kw_iterate.pop("reply_tag","iterate")
endpoint = "/spaces/{space_id}/iterate"
return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate)
def load_features_search(self, conn_info, **kw_iterate):
reply_tag = kw_iterate.pop("reply_tag","search")
endpoint = "/spaces/{space_id}/search"
return self._load_features_endpoint(endpoint, conn_info, reply_tag=reply_tag, **kw_iterate)
def _load_features_endpoint(self, endpoint, conn_info, reply_tag=None, **kw_iterate):
""" Iterate through all ordered features (no feature is repeated twice)
"""
kw_request = dict(kw_iterate)
kw_prop = dict(reply_tag=reply_tag)
kw_prop.update(kw_iterate)
return self._send_request(conn_info, endpoint, kw_request=kw_request, kw_prop=kw_prop)
###### feature function
def add_features(self, conn_info, added_feat, layer_id=None, **kw):
# POST, payload: list of FeatureCollection
endpoint = "/spaces/{space_id}/features"
kw_request = dict(req_type="geo", **kw) # kw: query
kw_prop = dict(reply_tag="add_feat",layer_id=layer_id)
kw_prop.update(kw)
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
buffer = make_payload(added_feat)
reply = self.network.post(request, buffer)
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
#parallel case (merge output ? split input?)
return reply
def del_features(self, conn_info, removed_feat, layer_id, **kw):
# DELETE by Query URL, required list of feat_id
query_del = {"id": ",".join(str(i) for i in removed_feat)}
kw.update(query_del)
endpoint = "/spaces/{space_id}/features"
kw_request = dict(kw) # kw: query
kw_prop = dict(reply_tag="del_feat",layer_id=layer_id)
request = self._pre_send_request(conn_info,endpoint,kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"DELETE")
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
def sync(self, conn_info, feat, layer_id, **kw):
added_feat, removed_feat = feat
token, space_id = conn_info.get_xyz_space()
if not added_feat is None:
self.add_features(conn_info, added_feat, layer_id)
if len(removed_feat):
self.del_features(conn_info, removed_feat, layer_id)
class KeyWordsSelection:
def __init__(self, auth_file):
logger.debug("\n================== ISOGEO API WITH QT ==================")
# getting credentials :
logger.debug("Getting credentials")
self.utils = IsogeoUtils()
self.app_creds = self.utils.credentials_loader(auth_file)
self.app_id = self.app_creds.get("client_id")
self.app_secrets = self.app_creds.get("client_secret")
# for connection :
self.naMngr = QNetworkAccessManager()
self.token_url = "https://id.api.isogeo.com/oauth/token"
self.request_url = (
"https://v1.api.isogeo.com/resources/search?_limit=0&_offset=0"
)
# init variables :
self.token = ""
self.search_type = "init"
self.checked_kw = {}
# for ui (launch and display):
logger.debug("Processing and displaying UI")
self.app = QApplication(sys.argv)
self.ui = AuthWidget()
self.ui.resize(400, 100)
self.pysdk_checking()
self.api_authentification()
self.ui.btn_reset.pressed.connect(self.reset)
self.ui.show()
self.app.exec()
def pysdk_checking(self):
logger.debug("\n--------------- isogeo-pysdk ---------------")
logger.debug("Checking credentials")
try:
isogeo = Isogeo(self.app_id, self.app_secrets)
isogeo.connect()
except OSError as e:
logger.debug("Credentials issue : {}".format(e))
return
except ValueError as e:
logger.debug("Credentials issue : {}".format(e))
return
if self.search_type == "init":
result = isogeo.search(
whole_share=0, page_size=0, augment=0, tags_as_dicts=1
)
else:
query = " ".join(list(self.checked_kw.keys()))
result = isogeo.search(
whole_share=0, page_size=0, augment=0, tags_as_dicts=1, query=query
)
self.tags_expected = result.get("tags")
self.kw_expected_nb = len(self.tags_expected.get("keywords"))
self.ui.lbl_expected.setText(
"Expected : {} resources and {} keywords".format(
result.get("total"), self.kw_expected_nb
)
)
logger.debug(
"isogeo-pysdk validates the authentication file, {} accessible resources".format(
result.get("total")
)
)
def api_authentification(self):
logger.debug("\n------------------ Authentication ------------------")
# creating credentials header
crd_header_value = QByteArray()
crd_header_value.append("Basic ")
crd_header_value.append(
base64.b64encode("{}:{}".format(self.app_id, self.app_secrets).encode())
)
crd_header_name = QByteArray()
crd_header_name.append("Authorization")
# creating Content-Type header
ct_header_value = QByteArray()
ct_header_value.append("application/json")
# creating request
token_rqst = QNetworkRequest(QUrl(self.token_url))
# setting headers
token_rqst.setRawHeader(crd_header_name, crd_header_value)
token_rqst.setHeader(token_rqst.ContentTypeHeader, ct_header_value)
# creating data
data = QByteArray()
data.append(urlencode({"grant_type": "client_credentials"}))
# requesting and handle reply
logger.debug("Asking for token")
token_reply = self.naMngr.post(token_rqst, data)
token_reply.finished.connect(partial(self.api_handle_token, reply=token_reply))
def api_handle_token(self, reply):
logger.debug("Token asked and API reply received")
# formating API response
bytarray = reply.readAll()
content = bytarray.data().decode("utf8")
# check API response structure
try:
parsed_content = json.loads(content)
except:
logger.debug("Reply format issue")
return
# check API response content
if "access_token" in parsed_content:
self.token = "Bearer " + parsed_content.get("access_token")
self.api_get_request()
else:
logger.debug("ya une couille dans la magouille : {}".format(parsed_content))
def api_get_request(self):
logger.debug("\n----------------- Sending request -----------------")
# creating credentials header
crd_header_value = QByteArray()
crd_header_value.append(self.token)
crd_header_name = QByteArray()
crd_header_name.append("Authorization")
# creating request
rqst = QNetworkRequest(QUrl(self.request_url))
# setting credentials header
rqst.setRawHeader(crd_header_name, crd_header_value) # sending request
rqst_reply = self.naMngr.get(rqst)
rqst_reply.finished.connect(partial(self.api_handle_request, reply=rqst_reply))
def api_handle_request(self, reply):
logger.debug("Request sent and API reply received")
bytarray = reply.readAll()
content = bytarray.data().decode("utf8")
if reply.error() == 0:
# check API response structure
try:
parsed_content = json.loads(content)
except:
logger.debug("Reply format issue")
return
# check API response content
self.tags_found = parsed_content.get("tags")
self.kw_found = {}
for tag in sorted(self.tags_found):
if tag.startswith("keyword:is"):
self.kw_found[tag] = self.tags_found.get(tag)
else:
pass
# displaying result
self.ui.lbl_found.setText(
"Found : {} resources and {} keywords".format(
parsed_content.get("total"), len(self.kw_found)
)
)
if self.search_type == "init":
if len(self.kw_found) == self.kw_expected_nb:
logger.debug("!!! It's working !!!")
else:
logger.debug("It's NOT working")
else:
pass
# filling keywords checkable combo box
self.pop_kw_cbbox()
elif self.search_type != "init":
logger.debug("token expired, renewing it")
self.api_authentification()
else:
pass
def pop_kw_cbbox(self):
logger.debug("\n-------------- Poping Keywords ComboBox --------------")
# to prepare the filling
self.ui.kw_cbbox.clear()
if self.search_type != "init":
self.ui.kw_cbbox.activated.disconnect(self.get_checked_kw)
# filling the combobox with checkable items
self.ui.kw_cbbox.addItem("-- Keywords --")
first_item = self.ui.kw_cbbox.model().item(0, 0)
first_item.setEnabled(False)
i = 1
for kw_code, kw_lbl in self.kw_found.items():
if self.search_type == "kw" and kw_code in self.checked_kw.keys():
self.ui.kw_cbbox.insertItem(1, kw_lbl)
item = self.ui.kw_cbbox.model().item(1, 0)
item.setCheckState(Qt.Checked)
else:
self.ui.kw_cbbox.addItem(kw_lbl)
item = self.ui.kw_cbbox.model().item(i, 0)
item.setCheckState(Qt.Unchecked)
item.setData(kw_code, 32)
i += 1
logger.debug("Keywords Combobox filled")
self.ui.kw_cbbox.setEnabled(True)
# connecting to a signal returning the checked item's index
self.ui.kw_cbbox.activated.connect(self.get_checked_kw)
def get_checked_kw(self, index):
logger.debug("\n------------ Getting Checked Keywords ------------")
self.ui.kw_cbbox.setEnabled(False)
self.ui.kw_cbbox.setCurrentText(self.ui.kw_cbbox.itemText(index))
# Testing if checked keyword is already in the dict is easier than
# testing if the user checked or unchecked it :
# removing the selected keyword from the dict if it is already in
if self.ui.kw_cbbox.itemData(index, 32) in self.checked_kw.keys():
del self.checked_kw[self.ui.kw_cbbox.itemData(index, 32)]
# adding the selected keyword to the dict if it is not already in
else:
self.checked_kw[
self.ui.kw_cbbox.itemData(index, 32)
] = self.ui.kw_cbbox.itemText(index)
logger.debug("ckeched kw : {}".format(self.checked_kw))
self.ui.lbl_selection.setText(
"{} keywords selected".format(len(self.checked_kw))
)
self.ui.kw_cbbox.setToolTip(" / ".join(list(self.checked_kw.values())))
# now selected keywords are stocked, time to request the API
self.kw_search()
def kw_search(self):
logger.debug("\n------------ Searching with keywords -------------")
# preparing the request
self.search_type = "kw"
self.request_url = self.url_builder()
self.pysdk_checking()
# launching the request
self.api_get_request()
def url_builder(self):
logger.debug("\n------------------ Building URL ------------------")
# adding selected keywords to the request URL
search_url = "https://v1.api.isogeo.com/resources/search?q="
for kw in self.checked_kw:
search_url += "{} ".format(str(kw))
logger.debug("URL : {}".format(search_url))
return search_url[:-1]
def reset(self):
logger.debug("----------------- RESET -------------------")
self.search_type = "reset"
self.checked_kw = {}
self.request_url = (
"https://v1.api.isogeo.com/resources/search?_limit=0&_offset=0"
)
self.ui.lbl_selection.setText("")
self.ui.kw_cbbox.setToolTip("")
self.pysdk_checking()
self.api_get_request()
class ApiConnection:
def __init__(self, auth_file):
logger.debug(
"\n================== ISOGEO API WITH QT ==================")
# getting credentials :
logger.debug("Getting credentials")
self.utils = IsogeoUtils()
self.app_creds = self.utils.credentials_loader(auth_file)
self.app_id = self.app_creds.get("client_id")
self.app_secrets = self.app_creds.get("client_secret")
# for connection :
self.naMngr = QNetworkAccessManager()
self.token_url = "https://id.api.isogeo.com/oauth/token"
self.request_url = (
"https://v1.api.isogeo.com/resources/search?_limit=0&_offset=0")
self.token = ""
# for ui :
logger.debug("Processing and displaying UI")
self.app = QApplication([])
self.ui = AuthWidget()
self.ui.resize(350, 100)
self.ui.btn.clicked.connect(self.api_authentification)
self.pysdk_checking()
self.ui.show()
self.app.exec()
def pysdk_checking(self):
logger.debug("\n------------------ isogeo-pysdk ------------------")
logger.debug("Checking credentials")
try:
isogeo = Isogeo(self.app_id, self.app_secrets)
isogeo.connect()
except OSError as e:
logger.debug("Credentials issue : {}".format(e))
return
except ValueError as e:
logger.debug("Credentials issue : {}".format(e))
return
self.md_expected = isogeo.search(whole_share=0, page_size=0,
augment=0).get("total")
self.ui.lbl_expected.setText("{} expected resources".format(
self.md_expected))
logger.debug(
"isogeo-pysdk validates the authentication file, {} accessible resources"
.format(self.md_expected))
def api_authentification(self):
logger.debug("\n------------------ Authentication ------------------")
# creating credentials header
logger.debug("Creating credentials header")
crd_header_value = QByteArray()
crd_header_value.append("Basic ")
crd_header_value.append(
base64.b64encode("{}:{}".format(self.app_id,
self.app_secrets).encode()))
crd_header_name = QByteArray()
crd_header_name.append("Authorization")
# creating Content-Type header
logger.debug("Creating 'Content-Type' header")
ct_header_value = QByteArray()
ct_header_value.append("application/json")
# creating request
token_rqst = QNetworkRequest(QUrl(self.token_url))
logger.debug("Creating token request : {}".format(token_rqst.url()))
# setting headers
token_rqst.setRawHeader(crd_header_name, crd_header_value)
logger.debug("Setting credentials header : {}".format(
token_rqst.rawHeader(crd_header_name)))
token_rqst.setHeader(token_rqst.ContentTypeHeader, ct_header_value)
logger.debug("Setting 'Content-Type' header : {}".format(
token_rqst.header(token_rqst.ContentTypeHeader)))
# creating data
data = QByteArray()
data.append(urlencode({"grant_type": "client_credentials"}))
logger.debug("Creating data : {}".format(data))
# requesting and handle reply
logger.debug("Asking for token")
token_reply = self.naMngr.post(token_rqst, data)
token_reply.finished.connect(
partial(self.api_handle_token, reply=token_reply))
def api_handle_token(self, reply):
logger.debug("\n------------------ Token retrieval ------------------")
logger.debug("Token asked and API reply received : {}".format(reply))
logger.debug("Storage and formatting of the reply")
bytarray = reply.readAll()
content = bytarray.data().decode("utf8")
# check API response structure
try:
parsed_content = json.loads(content)
except:
logger.debug("Reply format issue")
return
logger.debug("Reply format is good")
if "access_token" in parsed_content:
self.token = "Bearer " + parsed_content.get("access_token")
logger.debug("TOKEN STORED")
self.api_get_request()
else:
logger.debug(
"ya une couille dans la magouille : {}".format(parsed_content))
def api_get_request(self):
logger.debug("\n------------------ Sending request ------------------")
# creating credentials header
logger.debug("Creating credentials header")
crd_header_value = QByteArray()
crd_header_value.append(self.token)
crd_header_name = QByteArray()
crd_header_name.append("Authorization")
# creating request
rqst = QNetworkRequest(QUrl(self.request_url))
logger.debug("Creating request : {}".format(rqst.url()))
# setting credentials header
rqst.setRawHeader(crd_header_name, crd_header_value)
logger.debug("Setting credentials header : {}".format(
rqst.rawHeader(crd_header_name)))
# sending request
rqst_reply = self.naMngr.get(rqst)
logger.debug("Sending request")
rqst_reply.finished.connect(
partial(self.api_handle_request, reply=rqst_reply))
def api_handle_request(self, reply):
logger.debug("\n------------------ Reply retrieval ------------------")
logger.debug("Request sent and API reply received : {}".format(reply))
logger.debug("Storage and formatting of the reply")
bytarray = reply.readAll()
content = bytarray.data().decode("utf8")
# check API response structure
try:
parsed_content = json.loads(content)
except:
logger.debug("Reply format issue")
return
logger.debug("Reply format is good")
self.md_found = parsed_content.get("total")
logger.debug("RESULT STORED")
self.ui.lbl_found.setText("{} resources found".format(self.md_found))
if self.md_found == self.md_expected:
logger.debug("!!! It's working !!!")
else:
logger.debug("It's NOT working")
class NetManager(QObject):
def __init__(self, parent):
super().__init__(parent)
self.network = QNetworkAccessManager(self)
#############
def _pre_send_request(self, conn_info, endpoint, kw_request=dict()):
assert (isinstance(endpoint, str))
request = make_conn_request(conn_info, endpoint, **kw_request)
return request
def _post_send_request(self, reply, conn_info, kw_prop=dict()):
set_qt_property(reply, conn_info=conn_info, **kw_prop)
def _send_request(self,
conn_info,
endpoint,
kw_request=dict(),
kw_prop=dict()):
request = self._pre_send_request(conn_info,
endpoint,
kw_request=kw_request)
reply = self.network.get(request)
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
#############
# TODO: remove callback params
def get_statistics(self, conn_info):
reply = self._get_space_(conn_info, "statistics")
timeout = TIMEOUT_COUNT
QTimer.singleShot(timeout, reply.abort)
return reply
def get_count(self, conn_info):
reply = self._get_space_(conn_info, "count")
return reply
def get_meta(self, conn_info):
return self._get_space_(conn_info, "space_meta")
def _get_space_(self, conn_info, reply_tag):
tag = "/" + reply_tag if reply_tag != "space_meta" else ""
endpoint = "/spaces/{space_id}" + tag
kw_request = dict()
kw_prop = dict(reply_tag=reply_tag)
return self._send_request(conn_info,
endpoint,
kw_request=kw_request,
kw_prop=kw_prop)
def list_spaces(self, conn_info):
endpoint = "/spaces"
kw_request = dict(includeRights="true")
kw_prop = dict(reply_tag="spaces")
return self._send_request(conn_info,
endpoint,
kw_request=kw_request,
kw_prop=kw_prop)
def add_space(self, conn_info, space_info):
space_info = prepare_new_space_info(space_info)
endpoint = "/spaces"
kw_request = dict(req_type="json")
kw_prop = dict(reply_tag="add_space")
request = self._pre_send_request(conn_info,
endpoint,
kw_request=kw_request)
reply = self.network.post(request, make_payload(space_info))
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
def edit_space(self, conn_info, space_info):
endpoint = "/spaces/{space_id}"
kw_request = dict(req_type="json")
kw_prop = dict(reply_tag="edit_space")
request = self._pre_send_request(conn_info,
endpoint,
kw_request=kw_request)
buffer = make_buffer(space_info)
reply = self.network.sendCustomRequest(request, b"PATCH", buffer)
buffer.setParent(reply)
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
def del_space(self, conn_info):
endpoint = "/spaces/{space_id}"
kw_request = dict()
kw_prop = dict(reply_tag="del_space")
request = self._pre_send_request(conn_info,
endpoint,
kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"DELETE")
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
def _prefix_query(self, txt, prefix="p.", prefixes=tuple()):
""" return prefix to xyz query:
prefixes = ["p.", "f."] # add prefix if none in prefixes exists
prefixes = [] # always add prefix, dont check existing
"""
return prefix if not any(map(txt.startswith, prefixes)) else ""
def _process_queries(self, kw):
selection = ",".join(
"{prefix}{name}".format(name=p, prefix=self._prefix_query(p, "p."))
for p in kw.pop("selection", "").split(",") if p)
if selection: kw["selection"] = selection
self._process_raw_queries(kw)
def _process_raw_queries(self, kw):
filters = [
"{prefix}{name}{operator}{value}".format(name=p["name"],
operator=p["operator"],
value=p["values"],
prefix=self._prefix_query(
p["name"], "p."))
for p in kw.pop("filters", list())
]
if filters: kw.setdefault("raw_queries", list()).extend(filters)
def load_features_bbox(self, conn_info, bbox, **kw):
reply_tag = "bbox"
endpoint = "/spaces/{space_id}/bbox"
self._process_queries(kw)
kw_request = dict(bbox, **kw)
kw_prop = dict(reply_tag=reply_tag, bbox=bbox, **kw)
return self._send_request(conn_info,
endpoint,
kw_request=kw_request,
kw_prop=kw_prop)
def load_features_tile(self,
conn_info,
tile_id="0",
tile_schema="quadkey",
**kw):
reply_tag = "tile"
kw_tile = dict(tile_schema=tile_schema, tile_id=tile_id)
tile_url = "tile/{tile_schema}/{tile_id}".format(**kw_tile)
endpoint = "/spaces/{space_id}/" + tile_url
self._process_queries(kw)
kw_prop = dict(reply_tag=reply_tag, **dict(kw, **kw_tile))
return self._send_request(conn_info,
endpoint,
kw_request=kw,
kw_prop=kw_prop)
def load_features_iterate(self, conn_info, **kw):
reply_tag = kw.pop("reply_tag", "iterate")
endpoint = "/spaces/{space_id}/iterate"
self._process_queries(kw)
kw_prop = dict(reply_tag=reply_tag, **kw)
return self._send_request(conn_info,
endpoint,
kw_request=kw,
kw_prop=kw_prop)
def load_features_search(self, conn_info, **kw):
reply_tag = kw.pop("reply_tag", "search")
endpoint = "/spaces/{space_id}/search"
self._process_queries(kw)
kw_prop = dict(reply_tag=reply_tag, **kw)
return self._send_request(conn_info,
endpoint,
kw_request=kw,
kw_prop=kw_prop)
###### feature function
def add_features(self, conn_info, added_feat, **kw):
send_request = self.network.post # create or modify (merge existing feature with payload) # might add attributes
return self._add_features(conn_info, added_feat, send_request, **kw)
def modify_features(self, conn_info, added_feat, **kw):
return self.add_features(conn_info, added_feat, **kw)
def replace_features(self, conn_info, added_feat, **kw):
send_request = self.network.put # create or replace (replace existing feature with payload) # might add or drop attributes
return self._add_features(conn_info, added_feat, send_request, **kw)
def _add_features(self, conn_info, added_feat, send_request, **kw):
# POST, payload: list of FeatureCollection
endpoint = "/spaces/{space_id}/features"
if "tags" in kw: kw["addTags"] = kw["tags"]
kw_request = dict(req_type="geo", **kw) # kw: query
kw_prop = dict(reply_tag="add_feat")
kw_prop.update(kw)
request = self._pre_send_request(conn_info,
endpoint,
kw_request=kw_request)
payload = make_payload(added_feat)
reply = send_request(request, payload)
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
#parallel case (merge output ? split input?)
return reply
def del_features(self, conn_info, removed_feat, **kw):
# DELETE by Query URL, required list of feat_id
query_del = {"id": ",".join(str(i) for i in removed_feat)}
kw.update(query_del)
endpoint = "/spaces/{space_id}/features"
kw_request = dict(kw) # kw: query
kw_prop = dict(reply_tag="del_feat")
request = self._pre_send_request(conn_info,
endpoint,
kw_request=kw_request)
reply = self.network.sendCustomRequest(request, b"DELETE")
self._post_send_request(reply, conn_info, kw_prop=kw_prop)
return reply
class AccessSite(QObject):
abortReply = pyqtSignal()
finished = pyqtSignal(dict)
send_data = pyqtSignal(QByteArray)
status_download = pyqtSignal(int, int)
status_erros = pyqtSignal(list)
ErrorCodeAttribute = {
10: 'Canceled request',
400: 'Bad request syntax',
401: 'Unauthorized',
402: 'Payment required',
403: 'Forbidden',
404: 'Not found',
500: 'Internal error',
501: 'Not implemented',
502: 'Bad Gateway'
}
def __init__(self):
super().__init__()
self.isKill = None
self.responseAllFinished = None
self.nam = QNetworkAccessManager(self)
self.nam.finished.connect( self.replyFinished )
def _connectReply(self, reply, isConnect=True):
ss = [
{ 'signal': reply.readyRead, 'slot': self.readyRead },
{ 'signal': reply.sslErrors, 'slot': self.sslErrors }
]
if isConnect:
if not self.responseAllFinished:
reply.downloadProgress.connect( self.downloadProgress )
for item in ss:
item['signal'].connect( item['slot'] )
else:
if not self.responseAllFinished:
reply.downloadProgress.disconnect( self.downloadProgress )
for item in ss:
item['signal'].disconnect( item['slot'] )
def _closeReply(self, reply):
def connect(isConnect=True):
f_nam = self.nam.finished.connect if isConnect else self.nam.finished.disconnect
f_nam( self.replyFinished )
if not self.responseAllFinished:
self._connectReply( reply, isConnect )
# reply.close() call replyFinished
connect(False)
reply.close()
connect()
reply.deleteLater()
def _redirectionReply(self, reply, url):
self._closeReply( reply )
if url.isRelative():
url = url.resolved( url )
request = QNetworkRequest( url )
reply = self.nam.get( request )
if reply is None:
response = { 'isOk': False, 'message': "Netwok error", 'errorCode': -1 }
self.finished.emit( response )
return
if not self.responseAllFinished:
self._connectReply( reply )
def _emitErrorCodeAttribute(self, code, reply):
msg = 'Error network' if not code in self.ErrorCodeAttribute.keys() else self.ErrorCodeAttribute[ code ]
response = { 'isOk': False, 'message': msg, 'errorCode': code }
self._closeReply( reply )
self.finished.emit( response )
def _checkRedirectionAttribute(self, reply):
urlRedir = reply.attribute( QNetworkRequest.RedirectionTargetAttribute )
if not urlRedir is None and urlRedir != reply.url():
self._redirectionReply( reply, urlRedir )
return { 'isOk': True }
codeAttribute = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute )
if not ( 200 <= codeAttribute <= 299 ):
self._emitErrorCodeAttribute( codeAttribute, reply )
return { 'isOk': False }
return { 'isOk': True }
def _clearResponse(self, response):
if 'data' in response:
response['data'].clear()
del response[ 'data' ]
if 'statusRequest' in response:
del response['statusRequest']
if response['isOk']:
if 'errorCode' in response:
del response['errorCode']
def requestUrl(self, paramsAccess, addFinishedResponse, setFinished):
@pyqtSlot(dict)
def finished( response):
loop.quit()
self.finished.disconnect( finished )
if 'notResponseAllFinished' in paramsAccess:
self.send_data.disconnect( paramsAccess['notResponseAllFinished']['writePackageImage'] )
self.status_download.disconnect( paramsAccess['notResponseAllFinished']['progressPackageImage'] )
response = addFinishedResponse( response )
if response['isOk']:
self._clearResponse( response )
setFinished( response )
def run():
def sslVerifyNone(req):
conf = req.sslConfiguration()
conf.setPeerVerifyMode( QSslSocket.VerifyNone )
req.setSslConfiguration( conf )
req = QNetworkRequest( url )
sslVerifyNone( req ) # Need for Windows, error 'Handshake failed'
if json_request is None:
reply = self.nam.get( req )
else:
req.setHeader( QNetworkRequest.ContentTypeHeader, "application/json" )
data = QByteArray()
data.append( json.dumps( json_request ) )
reply = self.nam.post( req, data )
if reply is None:
response = { 'isOk': False, 'message': "Network error", 'errorCode': -1 }
self.finished.emit( response )
return
self.abortReply.connect( reply.abort )
if not self.responseAllFinished:
self._connectReply( reply )
loop = QEventLoop()
self.finished.connect( finished )
json_request = None if not 'json_request' in paramsAccess else paramsAccess['json_request']
self.responseAllFinished = True
if 'notResponseAllFinished' in paramsAccess:
self.responseAllFinished = False
self.send_data.connect( paramsAccess['notResponseAllFinished']['writePackageImage'] )
self.status_download.connect( paramsAccess['notResponseAllFinished']['progressPackageImage'] )
url = paramsAccess['url']
if 'credential' in paramsAccess:
userInfo = "{user}:{pwd}".format( user=paramsAccess['credential']['user'], pwd=paramsAccess['credential']['password'] )
url.setUserInfo( userInfo )
self.isKill = False
run()
loop.exec_()
def isHostLive(self, url, setFinished):
def addFinishedResponse(response):
if response['isOk']:
return response
else:
if response['errorCode'] == QNetworkReply.HostNotFoundError:
response['message'] = "{}\nURL = {}".format( response['message'], self.urlGeoserver )
else:
response['isOk'] = True
return response
p = { 'url': QUrl( url ) }
self.requestUrl( p, addFinishedResponse, setFinished )
def getThumbnail(self, url, setFinished):
def addFinishedResponse(response):
if not response['isOk']:
return response
if 'data' in response: # The user can quickly change a item
pixmap = QPixmap()
if not pixmap.loadFromData( response['data'] ):
response['isOk'] = False
response['message'] = 'Invalid image from Mapbiomas server'
else:
response['thumbnail'] = pixmap
return response
p = { 'url': QUrl( url ) }
self.requestUrl( p, addFinishedResponse, setFinished )
@pyqtSlot('QNetworkReply*')
def replyFinished(self, reply) :
if self.isKill:
self._emitErrorCodeAttribute(10, reply )
return
if reply.error() != QNetworkReply.NoError:
response = { 'isOk': False, 'message': reply.errorString(), 'errorCode': reply.error() }
self._closeReply( reply )
self.finished.emit( response )
return
r = self._checkRedirectionAttribute( reply )
if not r['isOk']:
return
statusRequest = {
'contentTypeHeader': reply.header( QNetworkRequest.ContentTypeHeader ),
'lastModifiedHeader': reply.header( QNetworkRequest.LastModifiedHeader ),
'contentLengthHeader': reply.header( QNetworkRequest.ContentLengthHeader ),
'statusCodeAttribute': reply.attribute( QNetworkRequest.HttpStatusCodeAttribute ),
'reasonPhraseAttribute': reply.attribute( QNetworkRequest.HttpReasonPhraseAttribute )
}
response = { 'isOk': True, 'statusRequest': statusRequest }
if self.responseAllFinished:
response['data'] = reply.readAll()
self._closeReply( reply )
self.finished.emit( response )
@pyqtSlot()
def readyRead(self):
reply = self.sender()
if self.isKill:
self._emitErrorCodeAttribute(10, reply )
return
r = self._checkRedirectionAttribute( reply )
if not r['isOk']:
return
if not reply.isOpen():
reply.open( QNetworkReply.ReadOnly )
data = reply.readAll()
if data is None:
return
self.send_data.emit( data )
@pyqtSlot('qint64', 'qint64')
def downloadProgress(self, bytesReceived, bytesTotal):
reply = self.sender()
if self.isKill:
self._emitErrorCodeAttribute(10, reply )
else:
self.status_download.emit( bytesReceived, bytesTotal )
@pyqtSlot('QList<QSslError>')
def sslErrors(self, errors):
reply = self.sender()
lstErros = map( lambda e: e.errorString(), errors )
self.status_erros.emit( lstErros )
reply.ignoreSslErrors()
@staticmethod
def loadJsonData(response):
data = response['data'].data()
sdata = str(data, encoding='utf-8')
return json.loads( sdata )