def send_transaction_request(**kwargs): pycsw_url = "http://meta.iguess.list.lu/" try: csw = CatalogueServiceWeb(pycsw_url) except: log_error_msg("Unable to create Catalogue object") text = "" try: with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "iguess", "csw_template.xml")), "r") as r: text = r.read() except: log_error_msg("problem reading the xml template") template = Template(text) try: result = template.render(**kwargs) except: log_error_msg("error rendering xml transaction template") try: csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=result) except: log_error_msg("catalogue record already present")
def deleteMetadata(name, login, password, url, workspace, verbose=False): #takes a keyword and delete metadata associated url_csw = url + "/geonetwork/srv/fre/csw-publication" keyword = workspace + ":" + name # keyword = geosync-restreint:baies_metadata__baies_metadata # Connect to a CSW, and inspect its properties: from owslib.csw import CatalogueServiceWeb csw = CatalogueServiceWeb(url_csw, skip_caps=True, username=login, password=password) # suppression des métadonnées relatives à "keyword" / couche geoserver # TODO faire sans owslib via API rest # ex: https://georchestra-mshe.univ-fcomte.fr/geonetwork/srv/fre/xml.search?any=geosync-ouvert:fouilles_chailluz__mobilier_pros # cette recherche est stricte au sens où cela trouve ce qui correspond exactement (exact match) par exemple la recherche de 'myworkspace:foo' ne trouve pas 'myworkspace:foobar' from owslib.fes import PropertyIsEqualTo, PropertyIsLike myquery = PropertyIsEqualTo( 'csw:AnyText', keyword ) # TODO vérifier que la recherche est stricte sinon risque de retourner des résultats non désirés (ex: 'myworkspace:foobar' en cherchant 'myworkspace:foo') if verbose: print "INFO CatalogueServiceWeb(" + url_csw + ",...) ... PropertyIsEqualTo('csw:AnyText'," + keyword + ")" csw.getrecords2(constraints=[myquery], maxrecords=10) resultat = csw.results if verbose: print "INFO " + str( csw.results['matches']) + " fiche(s) de metadata trouvée(s)" if csw.results['matches'] > 1: print "WARNING plus d'1 fiche de metadata trouvée pour " + keyword result = True # s'il y a au moins 1 erreur, retourne False for rec in csw.records: try: csw.transaction( ttype='delete', typename='MD_Metadata', identifier=csw.records[rec].identifier ) #marche apparement pour les metadonnees étant de type gmd:MD_Metadata result = True and result except Exception as e: sys.stderr.write("ERROR suppression de metadata pour " + keyword + " échouée : " + e.message + "\n") print "ERROR suppression de metadata pour " + keyword + " échouée" result = False try: identifier = csw.records[ rec].identifier #genere une erreur si pas d'identifiant # titre=csw.records[rec].title print "OK suppression de metadata " + keyword + " (" + identifier + ") réussie" except Exception as e: print "OK suppression de metadata réussie" print "WARNING " + e.message return result
def CSWInsert(CSW_URL, username, password, XML): """ Function to add a XML file to Geonetwork. Arguments: CSW_URL: The URL of the Geonetwork CSW publication server. username: The username to authenticate with. password: The password of the user. XML: The path to the XML file. """ try: csw = CatalogueServiceWeb(CSW_URL, skip_caps=True, username=username, password=password) csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open(XML).read()) log.debug('Inserted XML file to Geonetwork') except: log.error('Failed to add XML to Geonetwork') raise
def insert_metadata(gn_url, workspace, record, credentials=Credentials()): """ Inserts a metadata in a catalogue, given its base URL. :param gn_url: the base URL to the GeoNetwork. :param workspace: the workspace name queried onto the GeoServer :param record: the metadata as string to be inserted :param credentials: the credentials object :return: True if success, raises an exception otherwise. """ try: (username, password) = credentials.getFromUrl(gn_url) csw = CatalogueServiceWeb(gn_url + "/srv/eng/csw-publication", username=username, password=password) csw.transaction(ttype="insert", typename="gmd:MD_Metadata", record=record) return True except Exception as e: logger.error(e) raise GsToGnUnableToCreateServiceMetadataInconsistency( workspace, gn_url, e)
def update_metadata(gn_url, uuid, record, credentials=Credentials()): """ Updates a metadata in the catalogue, given its UUID :param gn_url the GeoNetwork base url :param uuid the unique identifier of the metadata to be updated :param record the XML as string for the updated metadata :param credentials the Credentials object :return: True if success, raises an exception otherwise """ try: (username, password) = credentials.getFromUrl(gn_url) csw = CatalogueServiceWeb(gn_url + "/srv/eng/csw-publication", username=username, password=password) csw.transaction(ttype="update", typename="gmd:MD_Metadata", record=record, identifier=uuid) return True except Exception as e: logger.error(e) raise GsToGnUnableToUpdateServiceMetadataInconsistency( workspace, uuid, gn_url, e)
class CSWOutput(BaseOutput): def __init__(self, config): self._csw_url = config.get(self.__class__.__name__, 'csw_url') self._csw_usr = config.get(self.__class__.__name__, 'csw_usr') if \ config.has_option(self.__class__.__name__, 'csw_usr') else None self._csw_pwd = config.get(self.__class__.__name__, 'csw_pwd') if \ config.has_option(self.__class__.__name__, 'csw_pwd') else None from owslib.csw import CatalogueServiceWeb self._csw = CatalogueServiceWeb(url=self._csw_url, skip_caps=True) @classmethod def type(cls): return 'CSW' def stream(self, buffer, record_id=None): self._csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=buffer) def __str__(self): return "CSW Output Stream"
def harvest_csw(src_url, dest_url): stop = 0 flag = 0 maxrecords = 10 src = CatalogueServiceWeb(src_url) dest = CatalogueServiceWeb(dest_url) while stop == 0: if flag == 0: # first run, start from 0 startposition = 0 else: # subsequent run, startposition is now paged startposition = src.results['nextrecord'] src.getrecords2(esn='full', startposition=startposition, maxrecords=maxrecords) print(src.results) if src.results['nextrecord'] == 0 \ or src.results['returned'] == 0 \ or src.results['nextrecord'] > src.results['matches']: # end the loop, exhausted all records stop = 1 break # harvest each record to destination CSW for i in list(src.records): print "insert", i src.getrecordbyid(id=[i], outputschema='http://www.isotc211.org/2005/gmd') md = src._exml.find('{http://www.isotc211.org/2005/gmd}MD_Metadata') f = open('/tmp/a.xml', 'w') f.write(etree.tostring(md)) f.close() dest.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open("/tmp/a.xml").read()) flag = 1
import sys import mapscript from owslib.csw import CatalogueServiceWeb #from checkbox.lib.text import split pycsw_url = "http://meta.iguess.list.lu/" if len(sys.argv) != 4: raise Exception("Script requires 3 args: dataset_url, dataset_identifier and dataset_id") scriptName, serverUrl, datasetIdentifier, datasetId = sys.argv try: id = "meta-" + str(datasetId) csw = CatalogueServiceWeb(pycsw_url) csw.transaction(ttype='delete', typename='gmd:MD_Metadata', identifier=id) except: print "Warning: transaction error while deleting the catalogue record" continue # Check the database to see if the server/identifier combo is still registered elsewhere try: # connect to iGUESS database conn=psycopg2.connect( "host={0} dbname={1} user={2} password={3}".format(dbServer, dbName, dbUsername, dbPassword)) cur = conn.cursor() cur.execute("SELECT d.id FROM iguess_dev.datasets d where d.server_url = '" + serverUrl + "' and d.identifier = '" + datasetIdentifier +"'") datasets = cur.fetchall() # Only if there is a single occurrence of the same combination of dataset identifier and server_url, # the dataset and layer in mapfile will be deleted if len(datasets) == 1:
class GeonetUserHandler(metaclass=Singleton): def __init__(self): self.username = GEONETWORK_LOGIN self.password = GEONETWORK_PASSWORD self.remote = CatalogueServiceWeb(urljoin(GEONETWORK_URL, 'srv/fre/csw-publication'), timeout=GEONETWORK_TIMEOUT, lang='fr-FR', version='2.0.2', skip_caps=True, username=self.username, password=self.password) def _get(self, url, params): r = requests.get(url, params=params, auth=(self.username, self.password)) r.raise_for_status() return r def _q(self, identifier): r = self._get(urljoin(GEONETWORK_URL, 'srv/fre/q'), { 'uuid': identifier, '_content_type': 'json' }) metadata = r.json().get('metadata') if metadata \ and len(metadata) == 1 \ and metadata[0]['uuid'] == identifier: return metadata[0]['id'] # Sinon error ? def _md_publish(self, identifier): return self._get(urljoin(GEONETWORK_URL, 'srv/fre/md.publish'), {'ids': identifier}) def _transaction(self, ttype, identifier, record=None): return self.remote.transaction(ttype=ttype, typename='gmd:MD_Metadata', identifier=identifier, record=record) def is_record_exists(self, id): return self.get_record(id) and True or False def get_record(self, id): try: self.remote.getrecordbyid( id=[id], outputschema='http://www.isotc211.org/2005/gmd') except requests.exceptions.HTTPError as e: logger.exception(e) if (e.response.status_code == 404): logger.warning("Error 404 was ignored.") return None raise e else: logger.debug("Get MD record with dc:identifier = '%s'" % id) return self.remote.records.get(id) def create_record(self, id, record): logger.debug("Create MD record with dc:identifier = '%s'" % id) return self._transaction('insert', id, record=record) def update_record(self, id, record): logger.debug("Update MD record with dc:identifier = '%s'" % id) return self._transaction('update', id, record=record) def delete_record(self, id): logger.debug("Delete MD record with dc:identifier = '%s'" % id) return self._transaction('delete', id) def publish(self, id): return self._md_publish(self._q(id))
def main(): print(f"Clearing Layman data.") # filesystem clear_directory(settings.LAYMAN_DATA_DIR) # qgis clear_directory(settings.LAYMAN_QGIS_DATA_DIR) # normalized raster data clear_directory(settings.LAYMAN_NORMALIZED_RASTER_DATA_DIR) # postgresql import psycopg2 conn_dict = settings.PG_CONN.copy() conn = psycopg2.connect(**settings.PG_CONN) conn.autocommit = True cur = conn.cursor() cur.execute(f""" select catalog_name, schema_name, schema_owner from information_schema.schemata where schema_owner = '{settings.LAYMAN_PG_USER}' and schema_name NOT IN ({', '.join(map(lambda s: "'" + s + "'", settings.PG_NON_USER_SCHEMAS))}) """) rows = cur.fetchall() print( f"Dropping schemas in DB {conn_dict['dbname']}: {[r[1] for r in rows]}" ) for row in rows: cur.execute(f"""DROP SCHEMA "{row[1]}" CASCADE""") conn.close() # redis print(f"Flushing Redis DB {settings.LAYMAN_REDIS_URL}") settings.LAYMAN_REDIS.flushdb() import redis print(f"Flushing Redis DB {os.environ['LTC_REDIS_URI']}") ltc_redis = redis.Redis.from_url(os.environ['LTC_REDIS_URI'], encoding="utf-8", decode_responses=True) ltc_redis.flushdb() # geoserver import requests headers_json = { 'Accept': 'application/json', 'Content-type': 'application/json', } auth = settings.GEOSERVER_ADMIN_AUTH or settings.LAYMAN_GS_AUTH response = requests.get(geoserver.GS_REST_USERS, headers=headers_json, auth=auth) response.raise_for_status() all_users = [u['userName'] for u in response.json()['users']] if settings.LAYMAN_GS_USER in all_users: all_users.remove(settings.LAYMAN_GS_USER) for user in all_users: response = requests.get(urljoin(geoserver.GS_REST_ROLES, f'user/{user}/'), headers=headers_json, auth=auth) response.raise_for_status() roles = response.json()['roles'] if settings.LAYMAN_GS_ROLE in roles: response = requests.delete(urljoin( geoserver.GS_REST_SECURITY_ACL_LAYERS, user + '.*.r'), headers=headers_json, auth=auth) if response.status_code != 404: response.raise_for_status() response = requests.delete(urljoin( geoserver.GS_REST_SECURITY_ACL_LAYERS, user + '.*.w'), headers=headers_json, auth=auth) if response.status_code != 404: response.raise_for_status() response = requests.delete(urljoin(geoserver.GS_REST_WORKSPACES, user), headers=headers_json, auth=auth, params={'recurse': 'true'}) response.raise_for_status() for role in roles: response = requests.delete( urljoin(geoserver.GS_REST_ROLES, f'role/{role}/user/{user}/'), headers=headers_json, auth=auth, ) response.raise_for_status() response = requests.delete( urljoin(geoserver.GS_REST_ROLES, 'role/' + f"USER_{user.upper()}"), headers=headers_json, auth=auth, ) if response.status_code != 404: response.raise_for_status() response = requests.delete( urljoin(geoserver.GS_REST_USER, user), headers=headers_json, auth=auth, ) response.raise_for_status() response = requests.get(geoserver.GS_REST_WORKSPACES, headers=headers_json, auth=auth) response.raise_for_status() if response.json()['workspaces'] != "": all_workspaces = [ workspace["name"] for workspace in response.json()['workspaces']['workspace'] ] for workspace in all_workspaces: response = requests.delete(urljoin(geoserver.GS_REST_WORKSPACES, workspace), headers=headers_json, auth=auth, params={'recurse': 'true'}) response.raise_for_status() # micka opts = {} if settings.CSW_BASIC_AUTHN is None else { 'username': settings.CSW_BASIC_AUTHN[0], 'password': settings.CSW_BASIC_AUTHN[1], } opts['skip_caps'] = True from owslib.csw import CatalogueServiceWeb csw = CatalogueServiceWeb(settings.CSW_URL, ** opts) if settings.CSW_URL is not None else None csw.getrecords2(xml=f""" <csw:GetRecords xmlns:ogc="http://www.opengis.net/ogc" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ows="http://www.opengis.net/ows" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0" xmlns:gmd="http://www.isotc211.org/2005/gmd" outputSchema="http://www.isotc211.org/2005/gmd" maxRecords="100" startPosition="1" outputFormat="application/xml" service="CSW" resultType="results" version="2.0.2" requestId="1" debug="0"> <csw:Query typeNames="gmd:MD_Metadata"> <csw:ElementSetName>summary</csw:ElementSetName> <csw:Constraint version="1.1.0"> <ogc:Filter xmlns:gml="http://www.opengis.net/gml"> <ogc:PropertyIsLike wildCard="*" singleChar="@" escapeChar="\\"> <ogc:PropertyName>apiso:Identifier</ogc:PropertyName> <ogc:Literal>*</ogc:Literal> </ogc:PropertyIsLike> </ogc:Filter> </csw:Constraint> </csw:Query> </csw:GetRecords> """) assert csw.exceptionreport is None items = csw.records.items() for record_id, record in items: urls = [ol.url for ol in record.distribution.online] url_part = f"://{settings.LAYMAN_PROXY_SERVER_NAME}/rest/" if any((url_part in u for u in urls)): print(f"Deleting metadata record {record_id}") csw.transaction(ttype='delete', typename='gmd:MD_Metadata', identifier=record_id) # Layman DB conn = psycopg2.connect(**settings.PG_CONN) conn.autocommit = True cur = conn.cursor() cur.execute(f""" DROP SCHEMA IF EXISTS "{settings.LAYMAN_PRIME_SCHEMA}" CASCADE; """) conn.commit()
class GeonetUserHandler(metaclass=Singleton): def __init__(self): self.username = GEONET_USERNAME self.password = GEONET_PASSWORD self.remote = CatalogueServiceWeb(urljoin(GEONET_URL, 'srv/fre/csw-publication'), timeout=GEONET_TIMEOUT, lang='fr-FR', version='2.0.2', skip_caps=True, username=self.username, password=self.password) def _get(self, url, params): r = requests.get(url, params=params, auth=(self.username, self.password)) r.raise_for_status() return r def _q(self, identifier): r = self._get(urljoin(GEONET_URL, 'srv/fre/q'), { 'uuid': identifier, '_content_type': 'json' }) metadata = r.json().get('metadata') if metadata \ and len(metadata) == 1 \ and metadata[0]['uuid'] == identifier: return metadata[0]['id'] # Sinon error ? def _md_publish(self, identifier): return self._get(urljoin(GEONET_URL, 'srv/fre/md.publish'), {'ids': identifier}) def _transaction(self, ttype, identifier, record): params = { 'identifier': identifier, 'record': record, 'ttype': ttype, 'typename': 'gmd:MD_Metadata' } return self.remote.transaction(**params) def is_record_exists(self, id): return self.get_record(id) and True or False def get_record(self, id): self.remote.getrecordbyid( id=[id], outputschema='http://www.isotc211.org/2005/gmd') return self.remote.records.get(id) def create_record(self, id, record): return self._transaction('insert', id, record) def update_record(self, id, record): return self._transaction('update', id, record) # def delete_record(self, id): # return self.remote.transaction('delete', id) def publish(self, id): return self._md_publish(self._q(id))
def publish_2_gn(input, url, login, password, workspace, verbose): from cleanName import cleanName import owslib print "dans metadata.publish_2_gn" print "Travail de CleanName.py" output = cleanName(input, True) print "Fin de CleanName.py" if verbose: print "input : ", input print "output : ", output print "url : ", url print "login : "******"password : "******"HOME"] # home = /home/georchestra-ouvert input = home + "/owncloudsync/" + input #print input import sys reload(sys) sys.setdefaultencoding('utf8') # création du répertoire temporaire tmpdir = home + "/tmp/geosync_metadata" if os.path.exists(tmpdir): import shutil shutil.rmtree(tmpdir) os.mkdir(tmpdir) else : os.mkdir(tmpdir) # Translate Esri metadata to ISO19139 #import codecs #file_input = codecs.open(input,'r',encoding='utf-8') # Force la reconnaissance du fichier en utf-8 file_input = open(input,'r') for line in file_input : if "<Esri>" in line : print "Métadonnée de type ArcGIS à convertir" # utilisation de saxonb pour traduire ArcGIS metadata => iso 19139 import subprocess saxon_input = "-s:" + input print str(saxon_input) saxon_xsl = "-xsl:" + home + "/bin/lib/ArcGIS2ISO19139.xsl" saxon_output = "-o:" + tmpdir + "/sax_" + output print str(saxon_output) subprocess.call(["saxonb-xslt", "-ext:on", saxon_input, saxon_xsl, saxon_output]) input = tmpdir + "/sax_" + output # input = /home/georchestra-ouvert/tmp/geosync_metadata/sax_cc_jura_nord.shp.xml print input break file_input.close() # Add Geoserver link to metadata and delete identifier from xml.dom import minidom doc = minidom.parse(input) root = doc.documentElement line1 = root.firstChild # Permet d'insérer des balises avec le namespace gmd line2 = line1.nextSibling.toprettyxml() # si le document xml original en contient line1 = line1.toprettyxml() # GN gère bien l'import avec ou sans gmd, du moment if ('gmd:' in line1) or ('gmd:' in line2) : # que le fichier soit cohérent print "Fichier avec gmd" gmd = 'gmd:' else : gmd = '' typename_csw = gmd + 'MD_Metadata' balise = gmd + 'fileIdentifier' for element in doc.getElementsByTagName(balise): print 'fileIdentifier en cours de suppression' doc.documentElement.removeChild(element) balise = gmd + 'MD_DigitalTransferOptions' test_digital = doc.getElementsByTagName(balise) if not test_digital : # creation de l'arborescence nécessaire à la creation de la balise MD_DigitalTransferOptions b_dist = gmd + 'distributionInfo' element_dist = doc.createElement(b_dist) b_MD_dist = gmd + 'MD_Distribution' element_MD_dist = doc.createElement(b_MD_dist) b_transfert = gmd + 'transferOptions' element_transfert = doc.createElement(b_transfert) b_digital = gmd + 'MD_DigitalTransferOptions' element_digital = doc.createElement(b_digital) for element in doc.getElementsByTagName(typename_csw) : print element.appendChild(element_dist) print element_dist.appendChild(element_MD_dist) print element_MD_dist.appendChild(element_transfert) print element_transfert.appendChild(element_digital) # AJOUT DES NOEUDS DANS L'ARBRE !!!! b_online = gmd + 'onLine' # creation balise online element_online = doc.createElement(b_online) b_ressource = gmd + 'CI_OnlineResource' # creation balise ressource element_ressource = doc.createElement(b_ressource) b_linkage = gmd + 'linkage' # creation balise linkage element_linkage = doc.createElement(b_linkage) b_url = gmd + 'URL' # creation et remplissage balise url element_url = doc.createElement(b_url) url_wms = url + "/geoserver/ows?SERVICE=WMS&" # url_wms = https://georchestra-dev.umrthema.univ-fcomte.fr/geoserver/ows?SERVICE=WMS& print url_wms element_url_txt = doc.createTextNode(url_wms) element_url.appendChild(element_url_txt) b_protocol = gmd + 'protocol' # creation et remplissage balise protocole element_protocol = doc.createElement(b_protocol) b_gco = 'gco:CharacterString' element_protocol_gco = doc.createElement(b_gco) element_protocol.appendChild(element_protocol_gco) #element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-capabilities") element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-map") element_protocol_gco.appendChild(element_protocol_txt) b_name = gmd + u'name' # creation et remplissage balise name element_name = doc.createElement(b_name) b_gco = 'gco:CharacterString' element_name_gco = doc.createElement(b_gco) element_name.appendChild(element_name_gco) name_layer_gs = workspace + ":" + output.split(".")[0] # name_layer_gs = geosync-restreint:baies_metadata__baies_metadata element_name_txt = doc.createTextNode(name_layer_gs) element_name_gco.appendChild(element_name_txt) b_descr = gmd +'description' # creation et remplissage balise description element_descr = doc.createElement(b_descr) b_gco = 'gco:CharacterString' element_descr_gco = doc.createElement(b_gco) element_descr.appendChild(element_descr_gco) element_descr_txt = doc.createTextNode(output.split(".")[0]) #baies_metadata__baies_metadata #u"baies_metadata__baies_metadataéééééééééé" element_descr_gco.appendChild(element_descr_txt) print balise for element in doc.getElementsByTagName(balise): print element.appendChild(element_online) print element_online.appendChild(element_ressource) print element_ressource.appendChild(element_linkage) print element_linkage.appendChild(element_url) print element_ressource.appendChild(element_protocol) print element_ressource.appendChild(element_name) print element_ressource.appendChild(element_descr) input_csw = tmpdir + "/csw_" + output output_fic = open(input_csw,'w') txt = doc.toxml().encode('utf-8','ignore') output_fic.write(txt) output_fic.close() # Connect to a CSW, and inspect its properties: from owslib.csw import CatalogueServiceWeb #csw = CatalogueServiceWeb(url, skip_caps=True, username=login, password=password) url_csw = url + "/geonetwork/srv/fre/csw-publication" csw = CatalogueServiceWeb(url_csw, skip_caps=True, username=login, password=password) #csw = CatalogueServiceWeb('https://georchestra-dev.umrthema.univ-fcomte.fr/geonetwork/srv/fre/csw-publication', skip_caps=True, username='******', password='******') # suppression des métadonnées relatives à la même couche geoserver from owslib.fes import PropertyIsEqualTo, PropertyIsLike myquery = PropertyIsEqualTo('csw:AnyText',name_layer_gs) csw.getrecords2(constraints=[myquery], maxrecords=10) resultat = csw.results print "resultat : " , resultat for rec in csw.records: print "suppression de " + csw.records[rec].title + csw.records[rec].identifier csw.transaction(ttype='delete', typename=typename_csw, identifier=csw.records[rec].identifier) # Transaction: insert #typename_csw = gmd + 'MD_Metadata' # FAIT PLUS HAUT, JUSTE APRES DETERMINATION GMD OU PAS #print typename_csw print input_csw #import unicodedata #acc = open(input_csw).read() #acc_8 = acc.decode('utf-8').encode('ascii', 'ignore') # supprime les caractères avec accents #acc_d = acc.decode('utf-8') #acc_8 = unicodedata.normalize('NFKD',acc_d).encode('ascii', 'ignore') # nécessite unicodedata, supprime les accents csw.transaction(ttype='insert', typename=typename_csw, record=open(input_csw).read()) #csw.transaction(ttype='insert', typename=typename_csw, record=acc_8) #csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open('haies_sans_lien_geoserver.xml').read()) # Update metadata privilege sql_req = "set schema 'geonetwork'; INSERT INTO operationallowed SELECT 1, metadata.id, 1 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; INSERT INTO operationallowed SELECT 1, metadata.id, 5 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ;" sql_file = open("update_privilege.sql","w") sql_file.write(sql_req) sql_file.close() os.system("psql -h localhost -d georchestra -U geosync -a -f update_privilege.sql")
pycsw_url = "http://meta.iguess.list.lu/" scriptname, service, identifier, city_id, abstract, server_url, title = sys.argv id = "meta-" + str(identifier) csw = CatalogueServiceWeb(pycsw_url) def serialize_metadata(**kwargs): text = "" with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "iguess", "csw_template.xml")), "r") as r: text = r.read() print r template = Template(text) result = template.render(**kwargs) return result now = datetime.datetime.now() organisation = "List" language = "eng" a = serialize_metadata( id=id, abstract=abstract, title=title, datestamp=now, organisation=organisation, language=language ) csw.transaction(ttype="insert", typename="gmd:MD_Metadata", record=a) print csw.results
# Search for keywords like ‘birds’ or ‘fowl’ birds_query_like = PropertyIsLike('dc:subject', '%birds%') fowl_query_like = PropertyIsLike('dc:subject', '%fowl%') csw.getrecords2(constraints=[birds_query_like, fowl_query_like]) csw.results # Search for a specific record: csw.getrecordbyid(id=['9250AA67-F3AC-6C12-0CB9-0662231AA181']) c.records['9250AA67-F3AC-6C12-0CB9-0662231AA181'].title # Search with a CQL query csw.getrecords(cql='csw:AnyText like "%birds%"') csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open("file.xml").read()) # update ALL records csw.transaction(ttype='update', typename='csw:Record', propertyname='dc:title', propertyvalue='New Title') # update records satisfying keywords filter csw.transaction(ttype='update', typename='csw:Record', propertyname='dc:title', propertyvalue='New Title', keywords=['birds', 'fowl']) # update records satisfying BBOX filter csw.transaction(ttype='update',
#try: # sends transaction-insert request as xml to pycsw server insert = requests.post(port, data=open(insertXml)) print(insert.content) #except Exception as e: # print("Error while inserting db entry. "+ str(e)) #insert Record with typename csw:Record else: uuid = root[0].text try: # sends transaction-insert request as xml to pycsw server csw = CatalogueServiceWeb(port) #print(urlopen(insertXml)) #print(urlopen(insertXml).read()) csw.transaction(ttype='insert', typename='csw:Record', record=xmlContent) #insert = requests.post(port, data = open(insertXml)) #print(insert.content) except Exception as e: print("Error while inserting db entry. " + str(e)) try: # sends transaciton-update request as xml to pycsw server update = requests.post(port, data=updateXml.createXmlTree(metadata, uuid)) print(update.content) except Exception as e: print("Error while updating db entry with id " + uuid + ". " + str(e))
class CSWConnexion(object): def __init__(self, csw_discovery_url, csw_publication_url, gn_username=None, gn_password=None): # TODO: make parameters for username and password self.getrecordbyidparam = '?request=GetRecordById&service=CSW&version=2.0.2&namespace=xmlns%28csw=http://www.opengis.net/cat/csw%29&resultType=results&outputSchema=http://www.isotc211.org/2005/gmd&outputFormat=application/xml&typeNames=csw:Record&elementSetName=full&constraintLanguage=CQL_TEXT&constraint_language_version=1.1.0' self.csw_discovery_url = csw_discovery_url self.csw_publication_url = csw_publication_url try: if gn_username is not None and gn_password is not None: self.csw_publication = CatalogueServiceWeb( self.csw_publication_url, username=gn_username, password=gn_password) self.csw_discovery = CatalogueServiceWeb( self.csw_discovery_url, username=gn_username, password=gn_password) except ServiceException: self.csw_publication = None self.csw_discovery = None try: self.csw_discovery = CatalogueServiceWeb(self.csw_discovery_url) except ServiceException: self.csw_discovery = None def get_record_list( self, keywords=[], typerecord=[] ): #par defaut les types et les keywords a selectionner sont vides --> toutes les fiches sont selectionnees from owslib.fes import PropertyIsEqualTo constraints = [] if len(typerecord) > 0: for type in typerecord: print(type) constraints.append(PropertyIsEqualTo('dc:type', type)) if len(keywords) > 0: for keyword in keywords: constraints.append(PropertyIsEqualTo('dc:subject', keyword)) constraints.append(PropertyIsEqualTo('keyword', keyword)) self.csw_discovery.getrecords2(constraints=constraints, maxrecords=100000) print(self.csw_discovery.results) self.identifiers = [] for rec in self.csw_discovery.records: self.identifiers.append(self.csw_discovery.records[rec].identifier) def updaterecord(self, id, input_xml): if self.csw_publication is not None: if isinstance(input_xml, etree._Element): input_xml = etree.tostring(input_xml) self.csw_publication.transaction(ttype='update', typename='gmd:MD_Metadata', record=input_xml, identifier=id) print("updated record " + id) else: print( "Could not perform operation as password-protected CSW is unreachable." ) def getrecordxml(self, id): """ fetches the metadata where the fileIdentifier is equal to id """ f = furl(self.csw_discovery_url + self.getrecordbyidparam) f.args['id'] = id r = http.request('GET', f.url) publishedrecord = r.data #publishedrecord=urlopen(f.url) tree = etree.fromstring(publishedrecord) #tree=etree.parse(str(publishedrecord)) root = tree[0] return root
def publish_2_gn(input, url, login, password, workspace, database_hostname, verbose): from cleanName import cleanName import owslib output = cleanName(input, True) if verbose: print "input : ", input print "output : ", output print "url : ", url print "login : "******"password : "******"workspace : ", workspace print "dbhost : ", database_hostname reload(sys) sys.setdefaultencoding('utf8') # aide au diagnostic : vérifie l'existence du fichier input if not os.path.isfile(input): sys.stderr.write("ERROR input file not found : " + input + "\n") return home = os.environ["HOME"] # création du répertoire temporaire tmpdir = home + "/tmp/geosync_metadata" if os.path.exists(tmpdir): import shutil shutil.rmtree(tmpdir, True) # ignore_errors try: os.mkdir(tmpdir) except OSError as e: if e.errno != errno.EEXIST: #sys.stderr.write("erreur lors de la création de tmpdir"+ tmpdir +"\n") raise # raises the error again else: os.mkdir(tmpdir) # Translate Esri metadata to ISO19139 # aide au diagnostic : vérifie la présence de ArcGIS2ISO19139.xsl script_path = os.path.dirname(os.path.abspath(__file__)) xsl_path = script_path + "/ArcGIS2ISO19139.xsl" if not os.path.isfile(xsl_path): sys.stderr.write("ERROR xsl file not found : " + xsl_path + "\n") return #import codecs #file_input = codecs.open(input,'r',encoding='utf-8') # Force la reconnaissance du fichier en utf-8 file_input = open(input, 'r') for line in file_input: if "<Esri>" in line: print "Métadonnée de type ArcGIS à convertir" # utilisation de saxonb pour traduire ArcGIS metadata => iso 19139 import subprocess saxon_input = "-s:" + input print str(saxon_input) saxon_xsl = "-xsl:" + xsl_path saxon_output = "-o:" + tmpdir + "/sax_" + output print str(saxon_output) # saxonb-xslt requiert le package libsaxonb-java (apt install libsaxonb-java) cmd = "saxonb-xslt", "-ext:on", saxon_input, saxon_xsl, saxon_output if verbose: print "saxonb cmd :", cmd subprocess.call(cmd) input = tmpdir + "/sax_" + output # input = /home/georchestra-ouvert/tmp/geosync_metadata/sax_cc_jura_nord.shp.xml print input break file_input.close() # Add Geoserver link to metadata and delete identifier # utilisation de lxml pour récupérer le contenu de la balise resTitle from lxml import etree tree = etree.parse(input) root = tree.getroot() resultat = root.xpath('//gmd:title/gco:CharacterString', namespaces={ 'gmd': 'http://www.isotc211.org/2005/gmd', 'gco': 'http://www.isotc211.org/2005/gco' }) titre = (resultat)[0] print "titre" print titre.text # utilisation de minidom pour modifier l'arbre xlm. à refaire avec lxml !! from xml.dom import minidom doc = minidom.parse(input) root = doc.documentElement line1 = root.firstChild # Permet d'insérer des balises avec le namespace gmd line2 = line1.nextSibling.toprettyxml( ) # si le document xml original en contient line1 = line1.toprettyxml( ) # geonetwork gère bien l'import avec ou sans gmd, du moment if ('gmd:' in line1) or ('gmd:' in line2): # que le fichier soit cohérent print "Fichier avec gmd" # GMD : Geographic MetaData extensible markup language gmd = 'gmd:' else: gmd = '' typename_csw = gmd + 'MD_Metadata' balise = gmd + 'fileIdentifier' for element in doc.getElementsByTagName(balise): print 'fileIdentifier en cours de suppression' doc.documentElement.removeChild(element) b_DigitalTransferOptions = gmd + 'MD_DigitalTransferOptions' test_digital = doc.getElementsByTagName(b_DigitalTransferOptions) if not test_digital: # creation de l'arborescence nécessaire à la creation de la balise MD_DigitalTransferOptions b_dist = gmd + 'distributionInfo' element_dist = doc.createElement(b_dist) b_MD_dist = gmd + 'MD_Distribution' element_MD_dist = doc.createElement(b_MD_dist) b_transfert = gmd + 'transferOptions' element_transfert = doc.createElement(b_transfert) b_digital = gmd + 'MD_DigitalTransferOptions' element_digital = doc.createElement(b_digital) for element in doc.getElementsByTagName(typename_csw): print element.appendChild(element_dist) print element_dist.appendChild(element_MD_dist) print element_MD_dist.appendChild(element_transfert) print element_transfert.appendChild(element_digital) # AJOUT DES NOEUDS DANS L'ARBRE # pour faire apparaître le bouton "Visualiser" dans geonetwork # dans <gmd:MD_DigitalTransferOptions> # <gmd:onLine> # <gmd:CI_OnlineResource> # <gmd:linkage> # <gmd:URL>https://georchestra-mshe.univ-fcomte.fr/geoserver/ows?SERVICE=WMS&</gmd:URL> # </gmd:linkage> # <gmd:protocol> # <gco:CharacterString>OGC:WMS-1.3.0-http-get-map</gco:CharacterString> # </gmd:protocol> # <gmd:name> # <gco:CharacterString>geosync-restreint:NOM_DE_LA_COUCHE</gco:CharacterString> # </gmd:name> # <gmd:description> # <gco:CharacterString>NOM_DE_LA_COUCHE</gco:CharacterString> # </gmd:description> # </gmd:CI_OnlineResource> # </gmd:onLine> b_online = gmd + 'onLine' # creation balise online element_online = doc.createElement(b_online) b_ressource = gmd + 'CI_OnlineResource' # creation balise ressource element_ressource = doc.createElement(b_ressource) b_linkage = gmd + 'linkage' # creation balise linkage element_linkage = doc.createElement(b_linkage) b_url = gmd + 'URL' # creation et remplissage balise url element_url = doc.createElement(b_url) url_wms = url + "/geoserver/ows?SERVICE=WMS&" # url_wms = https://georchestra-dev.umrthema.univ-fcomte.fr/geoserver/ows?SERVICE=WMS& print url_wms element_url_txt = doc.createTextNode(url_wms) element_url.appendChild(element_url_txt) b_protocol = gmd + 'protocol' # creation et remplissage balise protocole element_protocol = doc.createElement(b_protocol) b_gco = 'gco:CharacterString' element_protocol_gco = doc.createElement(b_gco) element_protocol.appendChild(element_protocol_gco) #element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-capabilities") element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-map") element_protocol_gco.appendChild(element_protocol_txt) b_name = gmd + u'name' # creation et remplissage balise name element_name = doc.createElement(b_name) b_gco = 'gco:CharacterString' element_name_gco = doc.createElement(b_gco) element_name.appendChild(element_name_gco) name_layer_gs = workspace + ":" + output.split(".")[ 0] # name_layer_gs = geosync-restreint:baies_metadata__baies_metadata element_name_txt = doc.createTextNode(name_layer_gs) element_name_gco.appendChild(element_name_txt) b_descr = gmd + 'description' # creation et remplissage balise description element_descr = doc.createElement(b_descr) b_gco = 'gco:CharacterString' print "b_gco " + b_gco element_descr_gco = doc.createElement(b_gco) element_descr.appendChild(element_descr_gco) element_descr_txt = doc.createTextNode( output.split(".")[0] ) #baies_metadata__baies_metadata #u"baies_metadata__baies_metadataéééééééééé" element_descr_gco.appendChild(element_descr_txt) print b_DigitalTransferOptions for element in doc.getElementsByTagName(b_DigitalTransferOptions): print element.appendChild(element_online) print element.toxml() print element_online.appendChild(element_ressource) print element.toxml() print element_ressource.appendChild(element_linkage) print element.toxml() print element_linkage.appendChild(element_url) print element.toxml() print element_ressource.appendChild(element_protocol) print element.toxml() print element_ressource.appendChild(element_name) print element.toxml() print element_ressource.appendChild(element_descr) print element.toxml() input_csw = tmpdir + "/csw_" + output output_fic = open(input_csw, 'w') txt = doc.toxml().encode('utf-8', 'ignore') output_fic.write(txt) output_fic.close() # Connect to a CSW, and inspect its properties: from owslib.csw import CatalogueServiceWeb #csw = CatalogueServiceWeb(url, skip_caps=True, username=login, password=password) url_csw = url + "/geonetwork/srv/fre/csw-publication" # Attention : l'utilisateur (login) doit avoir le rôle GN_EDITOR (ou GN_ADMIN) (anciennement SV_EDITOR / SV_ADMIN) voir administration ldap ## sinon peut générer l'erreur : lxml.etree.XMLSyntaxError: Opening and ending tag mismatch csw = CatalogueServiceWeb(url_csw, skip_caps=True, username=login, password=password) # suppression des métadonnées relatives à la même couche geoserver print "suppression de " + titre.text + " " + name_layer_gs from owslib.fes import PropertyIsEqualTo, PropertyIsLike myquery = PropertyIsEqualTo('csw:AnyText', name_layer_gs) csw.getrecords2(constraints=[myquery], maxrecords=10) resultat = csw.results print "resultat : ", resultat for rec in csw.records: print "suppression de " + csw.records[rec].title + csw.records[ rec].identifier csw.transaction(ttype='delete', typename=typename_csw, identifier=csw.records[rec].identifier) # Transaction: insert #typename_csw = gmd + 'MD_Metadata' # FAIT PLUS HAUT, JUSTE APRES DETERMINATION GMD OU PAS print "typename_csw " + typename_csw print "input_csw " + input_csw #import unicodedata #acc = open(input_csw).read() #acc_8 = acc.decode('utf-8').encode('ascii', 'ignore') # supprime les caractères avec accents #acc_d = acc.decode('utf-8') #acc_8 = unicodedata.normalize('NFKD',acc_d).encode('ascii', 'ignore') # nécessite unicodedata, supprime les accents csw.transaction(ttype='insert', typename=typename_csw, record=open(input_csw).read()) #csw.transaction(ttype='insert', typename=typename_csw, record=acc_8) #csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open('haies_sans_lien_geoserver.xml').read()) # Modification des privilèges d'une fiche de metadata pour pouvoir voir sa carte interactive / voir son lien de téléchargement des données : # doc : http://geonetwork-opensource.org/manuals/trunk/eng/users/user-guide/publishing/managing-privileges.html # via l'interface web : https://georchestra-mshe.univ-fcomte.fr/geonetwork/srv/fre/catalog.edit#/board # via la base de données (schema geonetwork) : # TODO il serait hautement préférable de passer par l'API REST que par la modification de la base de données de geonetwork # INSERT INTO geonetwork.operationallowed ( metadataid, groupid, operationid) VALUES ( '[METADATAID]', '1', '[OPERATIONID]'); # (groupid=1 pour "Tous") # * Télécharger (operationid=1) # * Carte interactive (operationid=5) # Update metadata privilege # sql_req = """set schema 'geonetwork'; # INSERT INTO operationallowed SELECT 1, metadata.id, 1 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; # INSERT INTO operationallowed SELECT 1, metadata.id, 5 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; # INSERT INTO operationallowed SELECT 1, metadata.id, 0 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' AND NOT EXISTS (SELECT * FROM operationallowed JOIN metadata ON operationallowed.metadataid = metadata.id WHERE data ILIKE '%" + name_layer_gs + "%' AND operationid = 0) ; """ # """ permette l'écriture sur plusieurs lignes" # -- attention -- # la recherche via "data ILIKE '%fouilles_chailluz__pt_limsit_sra%'" peut occasionner des doublons (par exemple avec un nom plus large qui inclut la chaîne cherchée, ou si la fiche de metadata a été duppliquée d'une autre sans être bien modifiée) # dans ce cas, produit l'erreur suivante : # psql:/tmp/geosync_metadata/update_privilege.sql:1: ERREUR: la valeur d'une clé dupliquée rompt la contrainte unique « operationallowed_pkey » # DÉTAIL : La clé « (groupid, metadataid, operationid)=(1, 485713, 1) » existe déjà. # l'erreur psql: fe_sendauth: no password supplied # peut être due à une erreur dans le user de la base de données de geonetwork, voir aussi le .pgpass sql_req = "set schema 'geonetwork'; INSERT INTO operationallowed SELECT 1, metadata.id, 5 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; INSERT INTO operationallowed SELECT 1, metadata.id, 0 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' AND NOT EXISTS (SELECT * FROM operationallowed JOIN metadata ON operationallowed.metadataid = metadata.id WHERE data ILIKE '%" + name_layer_gs + "%' AND operationid = 0) ; INSERT INTO operationallowed SELECT 1, metadata.id, 1 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; " print sql_req sql_file_name = tmpdir + "/update_privilege.sql" sql_file = open(sql_file_name, "w") sql_file.write(sql_req) sql_file.close() os.system("psql -h " + database_hostname + " -d georchestra -U geonetwork -w -a -f " + sql_file_name)