def send_transaction_request(**kwargs):
pycsw_url = "http://meta.iguess.list.lu/"
try:
csw = CatalogueServiceWeb(pycsw_url)
except:
log_error_msg("Unable to create Catalogue object")
text = ""
try:
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "iguess", "csw_template.xml")), "r") as r:
text = r.read()
except:
log_error_msg("problem reading the xml template")
template = Template(text)
try:
result = template.render(**kwargs)
except:
log_error_msg("error rendering xml transaction template")
try:
csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=result)
except:
log_error_msg("catalogue record already present")
def deleteMetadata(name, login, password, url, workspace, verbose=False):
#takes a keyword and delete metadata associated
url_csw = url + "/geonetwork/srv/fre/csw-publication"
keyword = workspace + ":" + name # keyword = geosync-restreint:baies_metadata__baies_metadata
# Connect to a CSW, and inspect its properties:
from owslib.csw import CatalogueServiceWeb
csw = CatalogueServiceWeb(url_csw,
skip_caps=True,
username=login,
password=password)
# suppression des métadonnées relatives à "keyword" / couche geoserver
# TODO faire sans owslib via API rest
# ex: https://georchestra-mshe.univ-fcomte.fr/geonetwork/srv/fre/xml.search?any=geosync-ouvert:fouilles_chailluz__mobilier_pros
# cette recherche est stricte au sens où cela trouve ce qui correspond exactement (exact match) par exemple la recherche de 'myworkspace:foo' ne trouve pas 'myworkspace:foobar'
from owslib.fes import PropertyIsEqualTo, PropertyIsLike
myquery = PropertyIsEqualTo(
'csw:AnyText', keyword
) # TODO vérifier que la recherche est stricte sinon risque de retourner des résultats non désirés (ex: 'myworkspace:foobar' en cherchant 'myworkspace:foo')
if verbose:
print "INFO CatalogueServiceWeb(" + url_csw + ",...) ... PropertyIsEqualTo('csw:AnyText'," + keyword + ")"
csw.getrecords2(constraints=[myquery], maxrecords=10)
resultat = csw.results
if verbose:
print "INFO " + str(
csw.results['matches']) + " fiche(s) de metadata trouvée(s)"
if csw.results['matches'] > 1:
print "WARNING plus d'1 fiche de metadata trouvée pour " + keyword
result = True # s'il y a au moins 1 erreur, retourne False
for rec in csw.records:
try:
csw.transaction(
ttype='delete',
typename='MD_Metadata',
identifier=csw.records[rec].identifier
) #marche apparement pour les metadonnees étant de type gmd:MD_Metadata
result = True and result
except Exception as e:
sys.stderr.write("ERROR suppression de metadata pour " + keyword +
" échouée : " + e.message + "\n")
print "ERROR suppression de metadata pour " + keyword + " échouée"
result = False
try:
identifier = csw.records[
rec].identifier #genere une erreur si pas d'identifiant
# titre=csw.records[rec].title
print "OK suppression de metadata " + keyword + " (" + identifier + ") réussie"
except Exception as e:
print "OK suppression de metadata réussie"
print "WARNING " + e.message
return result
def CSWInsert(CSW_URL, username, password, XML):
"""
Function to add a XML file to Geonetwork.
Arguments:
CSW_URL: The URL of the Geonetwork CSW publication server.
username: The username to authenticate with.
password: The password of the user.
XML: The path to the XML file.
"""
try:
csw = CatalogueServiceWeb(CSW_URL, skip_caps=True, username=username, password=password)
csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open(XML).read())
log.debug('Inserted XML file to Geonetwork')
except:
log.error('Failed to add XML to Geonetwork')
raise
def insert_metadata(gn_url, workspace, record, credentials=Credentials()):
"""
Inserts a metadata in a catalogue, given its base URL.
:param gn_url: the base URL to the GeoNetwork.
:param workspace: the workspace name queried onto the GeoServer
:param record: the metadata as string to be inserted
:param credentials: the credentials object
:return: True if success, raises an exception otherwise.
"""
try:
(username, password) = credentials.getFromUrl(gn_url)
csw = CatalogueServiceWeb(gn_url + "/srv/eng/csw-publication",
username=username,
password=password)
csw.transaction(ttype="insert",
typename="gmd:MD_Metadata",
record=record)
return True
except Exception as e:
logger.error(e)
raise GsToGnUnableToCreateServiceMetadataInconsistency(
workspace, gn_url, e)
def update_metadata(gn_url, uuid, record, credentials=Credentials()):
"""
Updates a metadata in the catalogue, given its UUID
:param gn_url the GeoNetwork base url
:param uuid the unique identifier of the metadata to be updated
:param record the XML as string for the updated metadata
:param credentials the Credentials object
:return: True if success, raises an exception otherwise
"""
try:
(username, password) = credentials.getFromUrl(gn_url)
csw = CatalogueServiceWeb(gn_url + "/srv/eng/csw-publication",
username=username,
password=password)
csw.transaction(ttype="update",
typename="gmd:MD_Metadata",
record=record,
identifier=uuid)
return True
except Exception as e:
logger.error(e)
raise GsToGnUnableToUpdateServiceMetadataInconsistency(
workspace, uuid, gn_url, e)
class CSWOutput(BaseOutput):
def __init__(self, config):
self._csw_url = config.get(self.__class__.__name__, 'csw_url')
self._csw_usr = config.get(self.__class__.__name__, 'csw_usr') if \
config.has_option(self.__class__.__name__, 'csw_usr') else None
self._csw_pwd = config.get(self.__class__.__name__, 'csw_pwd') if \
config.has_option(self.__class__.__name__, 'csw_pwd') else None
from owslib.csw import CatalogueServiceWeb
self._csw = CatalogueServiceWeb(url=self._csw_url, skip_caps=True)
@classmethod
def type(cls):
return 'CSW'
def stream(self, buffer, record_id=None):
self._csw.transaction(ttype='insert',
typename='gmd:MD_Metadata',
record=buffer)
def __str__(self):
return "CSW Output Stream"
def harvest_csw(src_url, dest_url):
stop = 0
flag = 0
maxrecords = 10
src = CatalogueServiceWeb(src_url)
dest = CatalogueServiceWeb(dest_url)
while stop == 0:
if flag == 0: # first run, start from 0
startposition = 0
else: # subsequent run, startposition is now paged
startposition = src.results['nextrecord']
src.getrecords2(esn='full', startposition=startposition, maxrecords=maxrecords)
print(src.results)
if src.results['nextrecord'] == 0 \
or src.results['returned'] == 0 \
or src.results['nextrecord'] > src.results['matches']: # end the loop, exhausted all records
stop = 1
break
# harvest each record to destination CSW
for i in list(src.records):
print "insert", i
src.getrecordbyid(id=[i], outputschema='http://www.isotc211.org/2005/gmd')
md = src._exml.find('{http://www.isotc211.org/2005/gmd}MD_Metadata')
f = open('/tmp/a.xml', 'w')
f.write(etree.tostring(md))
f.close()
dest.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open("/tmp/a.xml").read())
flag = 1
import sys
import mapscript
from owslib.csw import CatalogueServiceWeb
#from checkbox.lib.text import split
pycsw_url = "http://meta.iguess.list.lu/"
if len(sys.argv) != 4:
raise Exception("Script requires 3 args: dataset_url, dataset_identifier and dataset_id")
scriptName, serverUrl, datasetIdentifier, datasetId = sys.argv
try:
id = "meta-" + str(datasetId)
csw = CatalogueServiceWeb(pycsw_url)
csw.transaction(ttype='delete', typename='gmd:MD_Metadata', identifier=id)
except:
print "Warning: transaction error while deleting the catalogue record"
continue
# Check the database to see if the server/identifier combo is still registered elsewhere
try:
# connect to iGUESS database
conn=psycopg2.connect( "host={0} dbname={1} user={2} password={3}".format(dbServer, dbName, dbUsername, dbPassword))
cur = conn.cursor()
cur.execute("SELECT d.id FROM iguess_dev.datasets d where d.server_url = '" + serverUrl + "' and d.identifier = '" + datasetIdentifier +"'")
datasets = cur.fetchall()
# Only if there is a single occurrence of the same combination of dataset identifier and server_url,
# the dataset and layer in mapfile will be deleted
if len(datasets) == 1:
class GeonetUserHandler(metaclass=Singleton):
def __init__(self):
self.username = GEONETWORK_LOGIN
self.password = GEONETWORK_PASSWORD
self.remote = CatalogueServiceWeb(urljoin(GEONETWORK_URL,
'srv/fre/csw-publication'),
timeout=GEONETWORK_TIMEOUT,
lang='fr-FR',
version='2.0.2',
skip_caps=True,
username=self.username,
password=self.password)
def _get(self, url, params):
r = requests.get(url,
params=params,
auth=(self.username, self.password))
r.raise_for_status()
return r
def _q(self, identifier):
r = self._get(urljoin(GEONETWORK_URL, 'srv/fre/q'), {
'uuid': identifier,
'_content_type': 'json'
})
metadata = r.json().get('metadata')
if metadata \
and len(metadata) == 1 \
and metadata[0]['uuid'] == identifier:
return metadata[0]['id']
# Sinon error ?
def _md_publish(self, identifier):
return self._get(urljoin(GEONETWORK_URL, 'srv/fre/md.publish'),
{'ids': identifier})
def _transaction(self, ttype, identifier, record=None):
return self.remote.transaction(ttype=ttype,
typename='gmd:MD_Metadata',
identifier=identifier,
record=record)
def is_record_exists(self, id):
return self.get_record(id) and True or False
def get_record(self, id):
try:
self.remote.getrecordbyid(
id=[id], outputschema='http://www.isotc211.org/2005/gmd')
except requests.exceptions.HTTPError as e:
logger.exception(e)
if (e.response.status_code == 404):
logger.warning("Error 404 was ignored.")
return None
raise e
else:
logger.debug("Get MD record with dc:identifier = '%s'" % id)
return self.remote.records.get(id)
def create_record(self, id, record):
logger.debug("Create MD record with dc:identifier = '%s'" % id)
return self._transaction('insert', id, record=record)
def update_record(self, id, record):
logger.debug("Update MD record with dc:identifier = '%s'" % id)
return self._transaction('update', id, record=record)
def delete_record(self, id):
logger.debug("Delete MD record with dc:identifier = '%s'" % id)
return self._transaction('delete', id)
def publish(self, id):
return self._md_publish(self._q(id))
def main():
print(f"Clearing Layman data.")
# filesystem
clear_directory(settings.LAYMAN_DATA_DIR)
# qgis
clear_directory(settings.LAYMAN_QGIS_DATA_DIR)
# normalized raster data
clear_directory(settings.LAYMAN_NORMALIZED_RASTER_DATA_DIR)
# postgresql
import psycopg2
conn_dict = settings.PG_CONN.copy()
conn = psycopg2.connect(**settings.PG_CONN)
conn.autocommit = True
cur = conn.cursor()
cur.execute(f"""
select catalog_name, schema_name, schema_owner
from information_schema.schemata
where schema_owner = '{settings.LAYMAN_PG_USER}'
and schema_name NOT IN ({', '.join(map(lambda s: "'" + s + "'", settings.PG_NON_USER_SCHEMAS))})
""")
rows = cur.fetchall()
print(
f"Dropping schemas in DB {conn_dict['dbname']}: {[r[1] for r in rows]}"
)
for row in rows:
cur.execute(f"""DROP SCHEMA "{row[1]}" CASCADE""")
conn.close()
# redis
print(f"Flushing Redis DB {settings.LAYMAN_REDIS_URL}")
settings.LAYMAN_REDIS.flushdb()
import redis
print(f"Flushing Redis DB {os.environ['LTC_REDIS_URI']}")
ltc_redis = redis.Redis.from_url(os.environ['LTC_REDIS_URI'],
encoding="utf-8",
decode_responses=True)
ltc_redis.flushdb()
# geoserver
import requests
headers_json = {
'Accept': 'application/json',
'Content-type': 'application/json',
}
auth = settings.GEOSERVER_ADMIN_AUTH or settings.LAYMAN_GS_AUTH
response = requests.get(geoserver.GS_REST_USERS,
headers=headers_json,
auth=auth)
response.raise_for_status()
all_users = [u['userName'] for u in response.json()['users']]
if settings.LAYMAN_GS_USER in all_users:
all_users.remove(settings.LAYMAN_GS_USER)
for user in all_users:
response = requests.get(urljoin(geoserver.GS_REST_ROLES,
f'user/{user}/'),
headers=headers_json,
auth=auth)
response.raise_for_status()
roles = response.json()['roles']
if settings.LAYMAN_GS_ROLE in roles:
response = requests.delete(urljoin(
geoserver.GS_REST_SECURITY_ACL_LAYERS, user + '.*.r'),
headers=headers_json,
auth=auth)
if response.status_code != 404:
response.raise_for_status()
response = requests.delete(urljoin(
geoserver.GS_REST_SECURITY_ACL_LAYERS, user + '.*.w'),
headers=headers_json,
auth=auth)
if response.status_code != 404:
response.raise_for_status()
response = requests.delete(urljoin(geoserver.GS_REST_WORKSPACES,
user),
headers=headers_json,
auth=auth,
params={'recurse': 'true'})
response.raise_for_status()
for role in roles:
response = requests.delete(
urljoin(geoserver.GS_REST_ROLES,
f'role/{role}/user/{user}/'),
headers=headers_json,
auth=auth,
)
response.raise_for_status()
response = requests.delete(
urljoin(geoserver.GS_REST_ROLES,
'role/' + f"USER_{user.upper()}"),
headers=headers_json,
auth=auth,
)
if response.status_code != 404:
response.raise_for_status()
response = requests.delete(
urljoin(geoserver.GS_REST_USER, user),
headers=headers_json,
auth=auth,
)
response.raise_for_status()
response = requests.get(geoserver.GS_REST_WORKSPACES,
headers=headers_json,
auth=auth)
response.raise_for_status()
if response.json()['workspaces'] != "":
all_workspaces = [
workspace["name"]
for workspace in response.json()['workspaces']['workspace']
]
for workspace in all_workspaces:
response = requests.delete(urljoin(geoserver.GS_REST_WORKSPACES,
workspace),
headers=headers_json,
auth=auth,
params={'recurse': 'true'})
response.raise_for_status()
# micka
opts = {} if settings.CSW_BASIC_AUTHN is None else {
'username': settings.CSW_BASIC_AUTHN[0],
'password': settings.CSW_BASIC_AUTHN[1],
}
opts['skip_caps'] = True
from owslib.csw import CatalogueServiceWeb
csw = CatalogueServiceWeb(settings.CSW_URL, **
opts) if settings.CSW_URL is not None else None
csw.getrecords2(xml=f"""
<csw:GetRecords xmlns:ogc="http://www.opengis.net/ogc" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:ows="http://www.opengis.net/ows" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0" xmlns:gmd="http://www.isotc211.org/2005/gmd" outputSchema="http://www.isotc211.org/2005/gmd" maxRecords="100" startPosition="1" outputFormat="application/xml" service="CSW" resultType="results" version="2.0.2" requestId="1" debug="0">
<csw:Query typeNames="gmd:MD_Metadata">
<csw:ElementSetName>summary</csw:ElementSetName>
<csw:Constraint version="1.1.0">
<ogc:Filter xmlns:gml="http://www.opengis.net/gml">
<ogc:PropertyIsLike wildCard="*" singleChar="@" escapeChar="\\">
<ogc:PropertyName>apiso:Identifier</ogc:PropertyName>
<ogc:Literal>*</ogc:Literal>
</ogc:PropertyIsLike>
</ogc:Filter>
</csw:Constraint>
</csw:Query>
</csw:GetRecords>
""")
assert csw.exceptionreport is None
items = csw.records.items()
for record_id, record in items:
urls = [ol.url for ol in record.distribution.online]
url_part = f"://{settings.LAYMAN_PROXY_SERVER_NAME}/rest/"
if any((url_part in u for u in urls)):
print(f"Deleting metadata record {record_id}")
csw.transaction(ttype='delete',
typename='gmd:MD_Metadata',
identifier=record_id)
# Layman DB
conn = psycopg2.connect(**settings.PG_CONN)
conn.autocommit = True
cur = conn.cursor()
cur.execute(f"""
DROP SCHEMA IF EXISTS "{settings.LAYMAN_PRIME_SCHEMA}" CASCADE;
""")
conn.commit()
class GeonetUserHandler(metaclass=Singleton):
def __init__(self):
self.username = GEONET_USERNAME
self.password = GEONET_PASSWORD
self.remote = CatalogueServiceWeb(urljoin(GEONET_URL,
'srv/fre/csw-publication'),
timeout=GEONET_TIMEOUT,
lang='fr-FR',
version='2.0.2',
skip_caps=True,
username=self.username,
password=self.password)
def _get(self, url, params):
r = requests.get(url,
params=params,
auth=(self.username, self.password))
r.raise_for_status()
return r
def _q(self, identifier):
r = self._get(urljoin(GEONET_URL, 'srv/fre/q'), {
'uuid': identifier,
'_content_type': 'json'
})
metadata = r.json().get('metadata')
if metadata \
and len(metadata) == 1 \
and metadata[0]['uuid'] == identifier:
return metadata[0]['id']
# Sinon error ?
def _md_publish(self, identifier):
return self._get(urljoin(GEONET_URL, 'srv/fre/md.publish'),
{'ids': identifier})
def _transaction(self, ttype, identifier, record):
params = {
'identifier': identifier,
'record': record,
'ttype': ttype,
'typename': 'gmd:MD_Metadata'
}
return self.remote.transaction(**params)
def is_record_exists(self, id):
return self.get_record(id) and True or False
def get_record(self, id):
self.remote.getrecordbyid(
id=[id], outputschema='http://www.isotc211.org/2005/gmd')
return self.remote.records.get(id)
def create_record(self, id, record):
return self._transaction('insert', id, record)
def update_record(self, id, record):
return self._transaction('update', id, record)
# def delete_record(self, id):
# return self.remote.transaction('delete', id)
def publish(self, id):
return self._md_publish(self._q(id))
def publish_2_gn(input, url, login, password, workspace, verbose):
from cleanName import cleanName
import owslib
print "dans metadata.publish_2_gn"
print "Travail de CleanName.py"
output = cleanName(input, True)
print "Fin de CleanName.py"
if verbose:
print "input : ", input
print "output : ", output
print "url : ", url
print "login : "******"password : "******"HOME"] # home = /home/georchestra-ouvert
input = home + "/owncloudsync/" + input
#print input
import sys
reload(sys)
sys.setdefaultencoding('utf8')
# création du répertoire temporaire
tmpdir = home + "/tmp/geosync_metadata"
if os.path.exists(tmpdir):
import shutil
shutil.rmtree(tmpdir)
os.mkdir(tmpdir)
else :
os.mkdir(tmpdir)
# Translate Esri metadata to ISO19139
#import codecs
#file_input = codecs.open(input,'r',encoding='utf-8') # Force la reconnaissance du fichier en utf-8
file_input = open(input,'r')
for line in file_input :
if "<Esri>" in line :
print "Métadonnée de type ArcGIS à convertir"
# utilisation de saxonb pour traduire ArcGIS metadata => iso 19139
import subprocess
saxon_input = "-s:" + input
print str(saxon_input)
saxon_xsl = "-xsl:" + home + "/bin/lib/ArcGIS2ISO19139.xsl"
saxon_output = "-o:" + tmpdir + "/sax_" + output
print str(saxon_output)
subprocess.call(["saxonb-xslt", "-ext:on", saxon_input, saxon_xsl, saxon_output])
input = tmpdir + "/sax_" + output # input = /home/georchestra-ouvert/tmp/geosync_metadata/sax_cc_jura_nord.shp.xml
print input
break
file_input.close()
# Add Geoserver link to metadata and delete identifier
from xml.dom import minidom
doc = minidom.parse(input)
root = doc.documentElement
line1 = root.firstChild # Permet d'insérer des balises avec le namespace gmd
line2 = line1.nextSibling.toprettyxml() # si le document xml original en contient
line1 = line1.toprettyxml() # GN gère bien l'import avec ou sans gmd, du moment
if ('gmd:' in line1) or ('gmd:' in line2) : # que le fichier soit cohérent
print "Fichier avec gmd"
gmd = 'gmd:'
else :
gmd = ''
typename_csw = gmd + 'MD_Metadata'
balise = gmd + 'fileIdentifier'
for element in doc.getElementsByTagName(balise):
print 'fileIdentifier en cours de suppression'
doc.documentElement.removeChild(element)
balise = gmd + 'MD_DigitalTransferOptions'
test_digital = doc.getElementsByTagName(balise)
if not test_digital : # creation de l'arborescence nécessaire à la creation de la balise MD_DigitalTransferOptions
b_dist = gmd + 'distributionInfo'
element_dist = doc.createElement(b_dist)
b_MD_dist = gmd + 'MD_Distribution'
element_MD_dist = doc.createElement(b_MD_dist)
b_transfert = gmd + 'transferOptions'
element_transfert = doc.createElement(b_transfert)
b_digital = gmd + 'MD_DigitalTransferOptions'
element_digital = doc.createElement(b_digital)
for element in doc.getElementsByTagName(typename_csw) :
print element.appendChild(element_dist)
print element_dist.appendChild(element_MD_dist)
print element_MD_dist.appendChild(element_transfert)
print element_transfert.appendChild(element_digital)
# AJOUT DES NOEUDS DANS L'ARBRE !!!!
b_online = gmd + 'onLine' # creation balise online
element_online = doc.createElement(b_online)
b_ressource = gmd + 'CI_OnlineResource' # creation balise ressource
element_ressource = doc.createElement(b_ressource)
b_linkage = gmd + 'linkage' # creation balise linkage
element_linkage = doc.createElement(b_linkage)
b_url = gmd + 'URL' # creation et remplissage balise url
element_url = doc.createElement(b_url)
url_wms = url + "/geoserver/ows?SERVICE=WMS&" # url_wms = https://georchestra-dev.umrthema.univ-fcomte.fr/geoserver/ows?SERVICE=WMS&
print url_wms
element_url_txt = doc.createTextNode(url_wms)
element_url.appendChild(element_url_txt)
b_protocol = gmd + 'protocol' # creation et remplissage balise protocole
element_protocol = doc.createElement(b_protocol)
b_gco = 'gco:CharacterString'
element_protocol_gco = doc.createElement(b_gco)
element_protocol.appendChild(element_protocol_gco)
#element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-capabilities")
element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-map")
element_protocol_gco.appendChild(element_protocol_txt)
b_name = gmd + u'name' # creation et remplissage balise name
element_name = doc.createElement(b_name)
b_gco = 'gco:CharacterString'
element_name_gco = doc.createElement(b_gco)
element_name.appendChild(element_name_gco)
name_layer_gs = workspace + ":" + output.split(".")[0] # name_layer_gs = geosync-restreint:baies_metadata__baies_metadata
element_name_txt = doc.createTextNode(name_layer_gs)
element_name_gco.appendChild(element_name_txt)
b_descr = gmd +'description' # creation et remplissage balise description
element_descr = doc.createElement(b_descr)
b_gco = 'gco:CharacterString'
element_descr_gco = doc.createElement(b_gco)
element_descr.appendChild(element_descr_gco)
element_descr_txt = doc.createTextNode(output.split(".")[0]) #baies_metadata__baies_metadata #u"baies_metadata__baies_metadataéééééééééé"
element_descr_gco.appendChild(element_descr_txt)
print balise
for element in doc.getElementsByTagName(balise):
print element.appendChild(element_online)
print element_online.appendChild(element_ressource)
print element_ressource.appendChild(element_linkage)
print element_linkage.appendChild(element_url)
print element_ressource.appendChild(element_protocol)
print element_ressource.appendChild(element_name)
print element_ressource.appendChild(element_descr)
input_csw = tmpdir + "/csw_" + output
output_fic = open(input_csw,'w')
txt = doc.toxml().encode('utf-8','ignore')
output_fic.write(txt)
output_fic.close()
# Connect to a CSW, and inspect its properties:
from owslib.csw import CatalogueServiceWeb
#csw = CatalogueServiceWeb(url, skip_caps=True, username=login, password=password)
url_csw = url + "/geonetwork/srv/fre/csw-publication"
csw = CatalogueServiceWeb(url_csw, skip_caps=True, username=login, password=password)
#csw = CatalogueServiceWeb('https://georchestra-dev.umrthema.univ-fcomte.fr/geonetwork/srv/fre/csw-publication', skip_caps=True, username='******', password='******')
# suppression des métadonnées relatives à la même couche geoserver
from owslib.fes import PropertyIsEqualTo, PropertyIsLike
myquery = PropertyIsEqualTo('csw:AnyText',name_layer_gs)
csw.getrecords2(constraints=[myquery], maxrecords=10)
resultat = csw.results
print "resultat : " , resultat
for rec in csw.records:
print "suppression de " + csw.records[rec].title + csw.records[rec].identifier
csw.transaction(ttype='delete', typename=typename_csw, identifier=csw.records[rec].identifier)
# Transaction: insert
#typename_csw = gmd + 'MD_Metadata' # FAIT PLUS HAUT, JUSTE APRES DETERMINATION GMD OU PAS
#print typename_csw
print input_csw
#import unicodedata
#acc = open(input_csw).read()
#acc_8 = acc.decode('utf-8').encode('ascii', 'ignore') # supprime les caractères avec accents
#acc_d = acc.decode('utf-8')
#acc_8 = unicodedata.normalize('NFKD',acc_d).encode('ascii', 'ignore') # nécessite unicodedata, supprime les accents
csw.transaction(ttype='insert', typename=typename_csw, record=open(input_csw).read())
#csw.transaction(ttype='insert', typename=typename_csw, record=acc_8)
#csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open('haies_sans_lien_geoserver.xml').read())
# Update metadata privilege
sql_req = "set schema 'geonetwork'; INSERT INTO operationallowed SELECT 1, metadata.id, 1 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; INSERT INTO operationallowed SELECT 1, metadata.id, 5 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ;"
sql_file = open("update_privilege.sql","w")
sql_file.write(sql_req)
sql_file.close()
os.system("psql -h localhost -d georchestra -U geosync -a -f update_privilege.sql")
pycsw_url = "http://meta.iguess.list.lu/"
scriptname, service, identifier, city_id, abstract, server_url, title = sys.argv
id = "meta-" + str(identifier)
csw = CatalogueServiceWeb(pycsw_url)
def serialize_metadata(**kwargs):
text = ""
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "iguess", "csw_template.xml")), "r") as r:
text = r.read()
print r
template = Template(text)
result = template.render(**kwargs)
return result
now = datetime.datetime.now()
organisation = "List"
language = "eng"
a = serialize_metadata(
id=id, abstract=abstract, title=title, datestamp=now, organisation=organisation, language=language
)
csw.transaction(ttype="insert", typename="gmd:MD_Metadata", record=a)
print csw.results
# Search for keywords like ‘birds’ or ‘fowl’
birds_query_like = PropertyIsLike('dc:subject', '%birds%')
fowl_query_like = PropertyIsLike('dc:subject', '%fowl%')
csw.getrecords2(constraints=[birds_query_like, fowl_query_like])
csw.results
# Search for a specific record:
csw.getrecordbyid(id=['9250AA67-F3AC-6C12-0CB9-0662231AA181'])
c.records['9250AA67-F3AC-6C12-0CB9-0662231AA181'].title
# Search with a CQL query
csw.getrecords(cql='csw:AnyText like "%birds%"')
csw.transaction(ttype='insert',
typename='gmd:MD_Metadata',
record=open("file.xml").read())
# update ALL records
csw.transaction(ttype='update',
typename='csw:Record',
propertyname='dc:title',
propertyvalue='New Title')
# update records satisfying keywords filter
csw.transaction(ttype='update',
typename='csw:Record',
propertyname='dc:title',
propertyvalue='New Title',
keywords=['birds', 'fowl'])
# update records satisfying BBOX filter
csw.transaction(ttype='update',
#try:
# sends transaction-insert request as xml to pycsw server
insert = requests.post(port, data=open(insertXml))
print(insert.content)
#except Exception as e:
# print("Error while inserting db entry. "+ str(e))
#insert Record with typename csw:Record
else:
uuid = root[0].text
try:
# sends transaction-insert request as xml to pycsw server
csw = CatalogueServiceWeb(port)
#print(urlopen(insertXml))
#print(urlopen(insertXml).read())
csw.transaction(ttype='insert',
typename='csw:Record',
record=xmlContent)
#insert = requests.post(port, data = open(insertXml))
#print(insert.content)
except Exception as e:
print("Error while inserting db entry. " + str(e))
try:
# sends transaciton-update request as xml to pycsw server
update = requests.post(port, data=updateXml.createXmlTree(metadata, uuid))
print(update.content)
except Exception as e:
print("Error while updating db entry with id " + uuid + ". " + str(e))
class CSWConnexion(object):
def __init__(self,
csw_discovery_url,
csw_publication_url,
gn_username=None,
gn_password=None):
# TODO: make parameters for username and password
self.getrecordbyidparam = '?request=GetRecordById&service=CSW&version=2.0.2&namespace=xmlns%28csw=http://www.opengis.net/cat/csw%29&resultType=results&outputSchema=http://www.isotc211.org/2005/gmd&outputFormat=application/xml&typeNames=csw:Record&elementSetName=full&constraintLanguage=CQL_TEXT&constraint_language_version=1.1.0'
self.csw_discovery_url = csw_discovery_url
self.csw_publication_url = csw_publication_url
try:
if gn_username is not None and gn_password is not None:
self.csw_publication = CatalogueServiceWeb(
self.csw_publication_url,
username=gn_username,
password=gn_password)
self.csw_discovery = CatalogueServiceWeb(
self.csw_discovery_url,
username=gn_username,
password=gn_password)
except ServiceException:
self.csw_publication = None
self.csw_discovery = None
try:
self.csw_discovery = CatalogueServiceWeb(self.csw_discovery_url)
except ServiceException:
self.csw_discovery = None
def get_record_list(
self,
keywords=[],
typerecord=[]
): #par defaut les types et les keywords a selectionner sont vides --> toutes les fiches sont selectionnees
from owslib.fes import PropertyIsEqualTo
constraints = []
if len(typerecord) > 0:
for type in typerecord:
print(type)
constraints.append(PropertyIsEqualTo('dc:type', type))
if len(keywords) > 0:
for keyword in keywords:
constraints.append(PropertyIsEqualTo('dc:subject', keyword))
constraints.append(PropertyIsEqualTo('keyword', keyword))
self.csw_discovery.getrecords2(constraints=constraints,
maxrecords=100000)
print(self.csw_discovery.results)
self.identifiers = []
for rec in self.csw_discovery.records:
self.identifiers.append(self.csw_discovery.records[rec].identifier)
def updaterecord(self, id, input_xml):
if self.csw_publication is not None:
if isinstance(input_xml, etree._Element):
input_xml = etree.tostring(input_xml)
self.csw_publication.transaction(ttype='update',
typename='gmd:MD_Metadata',
record=input_xml,
identifier=id)
print("updated record " + id)
else:
print(
"Could not perform operation as password-protected CSW is unreachable."
)
def getrecordxml(self, id):
"""
fetches the metadata where the fileIdentifier is equal to id
"""
f = furl(self.csw_discovery_url + self.getrecordbyidparam)
f.args['id'] = id
r = http.request('GET', f.url)
publishedrecord = r.data
#publishedrecord=urlopen(f.url)
tree = etree.fromstring(publishedrecord)
#tree=etree.parse(str(publishedrecord))
root = tree[0]
return root
def publish_2_gn(input, url, login, password, workspace, database_hostname,
verbose):
from cleanName import cleanName
import owslib
output = cleanName(input, True)
if verbose:
print "input : ", input
print "output : ", output
print "url : ", url
print "login : "******"password : "******"workspace : ", workspace
print "dbhost : ", database_hostname
reload(sys)
sys.setdefaultencoding('utf8')
# aide au diagnostic : vérifie l'existence du fichier input
if not os.path.isfile(input):
sys.stderr.write("ERROR input file not found : " + input + "\n")
return
home = os.environ["HOME"]
# création du répertoire temporaire
tmpdir = home + "/tmp/geosync_metadata"
if os.path.exists(tmpdir):
import shutil
shutil.rmtree(tmpdir, True) # ignore_errors
try:
os.mkdir(tmpdir)
except OSError as e:
if e.errno != errno.EEXIST:
#sys.stderr.write("erreur lors de la création de tmpdir"+ tmpdir +"\n")
raise # raises the error again
else:
os.mkdir(tmpdir)
# Translate Esri metadata to ISO19139
# aide au diagnostic : vérifie la présence de ArcGIS2ISO19139.xsl
script_path = os.path.dirname(os.path.abspath(__file__))
xsl_path = script_path + "/ArcGIS2ISO19139.xsl"
if not os.path.isfile(xsl_path):
sys.stderr.write("ERROR xsl file not found : " + xsl_path + "\n")
return
#import codecs
#file_input = codecs.open(input,'r',encoding='utf-8') # Force la reconnaissance du fichier en utf-8
file_input = open(input, 'r')
for line in file_input:
if "<Esri>" in line:
print "Métadonnée de type ArcGIS à convertir"
# utilisation de saxonb pour traduire ArcGIS metadata => iso 19139
import subprocess
saxon_input = "-s:" + input
print str(saxon_input)
saxon_xsl = "-xsl:" + xsl_path
saxon_output = "-o:" + tmpdir + "/sax_" + output
print str(saxon_output)
# saxonb-xslt requiert le package libsaxonb-java (apt install libsaxonb-java)
cmd = "saxonb-xslt", "-ext:on", saxon_input, saxon_xsl, saxon_output
if verbose:
print "saxonb cmd :", cmd
subprocess.call(cmd)
input = tmpdir + "/sax_" + output # input = /home/georchestra-ouvert/tmp/geosync_metadata/sax_cc_jura_nord.shp.xml
print input
break
file_input.close()
# Add Geoserver link to metadata and delete identifier
# utilisation de lxml pour récupérer le contenu de la balise resTitle
from lxml import etree
tree = etree.parse(input)
root = tree.getroot()
resultat = root.xpath('//gmd:title/gco:CharacterString',
namespaces={
'gmd': 'http://www.isotc211.org/2005/gmd',
'gco': 'http://www.isotc211.org/2005/gco'
})
titre = (resultat)[0]
print "titre"
print titre.text
# utilisation de minidom pour modifier l'arbre xlm. à refaire avec lxml !!
from xml.dom import minidom
doc = minidom.parse(input)
root = doc.documentElement
line1 = root.firstChild # Permet d'insérer des balises avec le namespace gmd
line2 = line1.nextSibling.toprettyxml(
) # si le document xml original en contient
line1 = line1.toprettyxml(
) # geonetwork gère bien l'import avec ou sans gmd, du moment
if ('gmd:' in line1) or ('gmd:' in line2): # que le fichier soit cohérent
print "Fichier avec gmd" # GMD : Geographic MetaData extensible markup language
gmd = 'gmd:'
else:
gmd = ''
typename_csw = gmd + 'MD_Metadata'
balise = gmd + 'fileIdentifier'
for element in doc.getElementsByTagName(balise):
print 'fileIdentifier en cours de suppression'
doc.documentElement.removeChild(element)
b_DigitalTransferOptions = gmd + 'MD_DigitalTransferOptions'
test_digital = doc.getElementsByTagName(b_DigitalTransferOptions)
if not test_digital: # creation de l'arborescence nécessaire à la creation de la balise MD_DigitalTransferOptions
b_dist = gmd + 'distributionInfo'
element_dist = doc.createElement(b_dist)
b_MD_dist = gmd + 'MD_Distribution'
element_MD_dist = doc.createElement(b_MD_dist)
b_transfert = gmd + 'transferOptions'
element_transfert = doc.createElement(b_transfert)
b_digital = gmd + 'MD_DigitalTransferOptions'
element_digital = doc.createElement(b_digital)
for element in doc.getElementsByTagName(typename_csw):
print element.appendChild(element_dist)
print element_dist.appendChild(element_MD_dist)
print element_MD_dist.appendChild(element_transfert)
print element_transfert.appendChild(element_digital)
# AJOUT DES NOEUDS DANS L'ARBRE
# pour faire apparaître le bouton "Visualiser" dans geonetwork
# dans <gmd:MD_DigitalTransferOptions>
# <gmd:onLine>
# <gmd:CI_OnlineResource>
# <gmd:linkage>
# <gmd:URL>https://georchestra-mshe.univ-fcomte.fr/geoserver/ows?SERVICE=WMS&</gmd:URL>
# </gmd:linkage>
# <gmd:protocol>
# <gco:CharacterString>OGC:WMS-1.3.0-http-get-map</gco:CharacterString>
# </gmd:protocol>
# <gmd:name>
# <gco:CharacterString>geosync-restreint:NOM_DE_LA_COUCHE</gco:CharacterString>
# </gmd:name>
# <gmd:description>
# <gco:CharacterString>NOM_DE_LA_COUCHE</gco:CharacterString>
# </gmd:description>
# </gmd:CI_OnlineResource>
# </gmd:onLine>
b_online = gmd + 'onLine' # creation balise online
element_online = doc.createElement(b_online)
b_ressource = gmd + 'CI_OnlineResource' # creation balise ressource
element_ressource = doc.createElement(b_ressource)
b_linkage = gmd + 'linkage' # creation balise linkage
element_linkage = doc.createElement(b_linkage)
b_url = gmd + 'URL' # creation et remplissage balise url
element_url = doc.createElement(b_url)
url_wms = url + "/geoserver/ows?SERVICE=WMS&" # url_wms = https://georchestra-dev.umrthema.univ-fcomte.fr/geoserver/ows?SERVICE=WMS&
print url_wms
element_url_txt = doc.createTextNode(url_wms)
element_url.appendChild(element_url_txt)
b_protocol = gmd + 'protocol' # creation et remplissage balise protocole
element_protocol = doc.createElement(b_protocol)
b_gco = 'gco:CharacterString'
element_protocol_gco = doc.createElement(b_gco)
element_protocol.appendChild(element_protocol_gco)
#element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-capabilities")
element_protocol_txt = doc.createTextNode(u"OGC:WMS-1.3.0-http-get-map")
element_protocol_gco.appendChild(element_protocol_txt)
b_name = gmd + u'name' # creation et remplissage balise name
element_name = doc.createElement(b_name)
b_gco = 'gco:CharacterString'
element_name_gco = doc.createElement(b_gco)
element_name.appendChild(element_name_gco)
name_layer_gs = workspace + ":" + output.split(".")[
0] # name_layer_gs = geosync-restreint:baies_metadata__baies_metadata
element_name_txt = doc.createTextNode(name_layer_gs)
element_name_gco.appendChild(element_name_txt)
b_descr = gmd + 'description' # creation et remplissage balise description
element_descr = doc.createElement(b_descr)
b_gco = 'gco:CharacterString'
print "b_gco " + b_gco
element_descr_gco = doc.createElement(b_gco)
element_descr.appendChild(element_descr_gco)
element_descr_txt = doc.createTextNode(
output.split(".")[0]
) #baies_metadata__baies_metadata #u"baies_metadata__baies_metadataéééééééééé"
element_descr_gco.appendChild(element_descr_txt)
print b_DigitalTransferOptions
for element in doc.getElementsByTagName(b_DigitalTransferOptions):
print element.appendChild(element_online)
print element.toxml()
print element_online.appendChild(element_ressource)
print element.toxml()
print element_ressource.appendChild(element_linkage)
print element.toxml()
print element_linkage.appendChild(element_url)
print element.toxml()
print element_ressource.appendChild(element_protocol)
print element.toxml()
print element_ressource.appendChild(element_name)
print element.toxml()
print element_ressource.appendChild(element_descr)
print element.toxml()
input_csw = tmpdir + "/csw_" + output
output_fic = open(input_csw, 'w')
txt = doc.toxml().encode('utf-8', 'ignore')
output_fic.write(txt)
output_fic.close()
# Connect to a CSW, and inspect its properties:
from owslib.csw import CatalogueServiceWeb
#csw = CatalogueServiceWeb(url, skip_caps=True, username=login, password=password)
url_csw = url + "/geonetwork/srv/fre/csw-publication"
# Attention : l'utilisateur (login) doit avoir le rôle GN_EDITOR (ou GN_ADMIN) (anciennement SV_EDITOR / SV_ADMIN) voir administration ldap
## sinon peut générer l'erreur : lxml.etree.XMLSyntaxError: Opening and ending tag mismatch
csw = CatalogueServiceWeb(url_csw,
skip_caps=True,
username=login,
password=password)
# suppression des métadonnées relatives à la même couche geoserver
print "suppression de " + titre.text + " " + name_layer_gs
from owslib.fes import PropertyIsEqualTo, PropertyIsLike
myquery = PropertyIsEqualTo('csw:AnyText', name_layer_gs)
csw.getrecords2(constraints=[myquery], maxrecords=10)
resultat = csw.results
print "resultat : ", resultat
for rec in csw.records:
print "suppression de " + csw.records[rec].title + csw.records[
rec].identifier
csw.transaction(ttype='delete',
typename=typename_csw,
identifier=csw.records[rec].identifier)
# Transaction: insert
#typename_csw = gmd + 'MD_Metadata' # FAIT PLUS HAUT, JUSTE APRES DETERMINATION GMD OU PAS
print "typename_csw " + typename_csw
print "input_csw " + input_csw
#import unicodedata
#acc = open(input_csw).read()
#acc_8 = acc.decode('utf-8').encode('ascii', 'ignore') # supprime les caractères avec accents
#acc_d = acc.decode('utf-8')
#acc_8 = unicodedata.normalize('NFKD',acc_d).encode('ascii', 'ignore') # nécessite unicodedata, supprime les accents
csw.transaction(ttype='insert',
typename=typename_csw,
record=open(input_csw).read())
#csw.transaction(ttype='insert', typename=typename_csw, record=acc_8)
#csw.transaction(ttype='insert', typename='gmd:MD_Metadata', record=open('haies_sans_lien_geoserver.xml').read())
# Modification des privilèges d'une fiche de metadata pour pouvoir voir sa carte interactive / voir son lien de téléchargement des données :
# doc : http://geonetwork-opensource.org/manuals/trunk/eng/users/user-guide/publishing/managing-privileges.html
# via l'interface web : https://georchestra-mshe.univ-fcomte.fr/geonetwork/srv/fre/catalog.edit#/board
# via la base de données (schema geonetwork) :
# TODO il serait hautement préférable de passer par l'API REST que par la modification de la base de données de geonetwork
# INSERT INTO geonetwork.operationallowed ( metadataid, groupid, operationid) VALUES ( '[METADATAID]', '1', '[OPERATIONID]');
# (groupid=1 pour "Tous")
# * Télécharger (operationid=1)
# * Carte interactive (operationid=5)
# Update metadata privilege
# sql_req = """set schema 'geonetwork';
# INSERT INTO operationallowed SELECT 1, metadata.id, 1 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ;
# INSERT INTO operationallowed SELECT 1, metadata.id, 5 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ;
# INSERT INTO operationallowed SELECT 1, metadata.id, 0 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' AND NOT EXISTS (SELECT * FROM operationallowed JOIN metadata ON operationallowed.metadataid = metadata.id WHERE data ILIKE '%" + name_layer_gs + "%' AND operationid = 0) ; """ # """ permette l'écriture sur plusieurs lignes"
# -- attention --
# la recherche via "data ILIKE '%fouilles_chailluz__pt_limsit_sra%'" peut occasionner des doublons (par exemple avec un nom plus large qui inclut la chaîne cherchée, ou si la fiche de metadata a été duppliquée d'une autre sans être bien modifiée)
# dans ce cas, produit l'erreur suivante :
# psql:/tmp/geosync_metadata/update_privilege.sql:1: ERREUR: la valeur d'une clé dupliquée rompt la contrainte unique « operationallowed_pkey »
# DÉTAIL : La clé « (groupid, metadataid, operationid)=(1, 485713, 1) » existe déjà.
# l'erreur psql: fe_sendauth: no password supplied
# peut être due à une erreur dans le user de la base de données de geonetwork, voir aussi le .pgpass
sql_req = "set schema 'geonetwork'; INSERT INTO operationallowed SELECT 1, metadata.id, 5 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; INSERT INTO operationallowed SELECT 1, metadata.id, 0 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' AND NOT EXISTS (SELECT * FROM operationallowed JOIN metadata ON operationallowed.metadataid = metadata.id WHERE data ILIKE '%" + name_layer_gs + "%' AND operationid = 0) ; INSERT INTO operationallowed SELECT 1, metadata.id, 1 FROM metadata WHERE data ILIKE '%" + name_layer_gs + "%' ; "
print sql_req
sql_file_name = tmpdir + "/update_privilege.sql"
sql_file = open(sql_file_name, "w")
sql_file.write(sql_req)
sql_file.close()
os.system("psql -h " + database_hostname +
" -d georchestra -U geonetwork -w -a -f " + sql_file_name)
