# Create the web service proxy
config = getConfig()
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL()
servicePort = config.getint('DEFAULT','hessian_service_port')
serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug')
servicePollingDelay = config.getfloat('DEFAULT','hessian_service_polling_delay')
spi = servicePollingIterations = config.getint('DEFAULT','hessian_service_polling_iterations')
service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug)
else: # REST service
spi = 1
serviceURL = getRestServiceURL()
serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False)
service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug)
results = []
lenresults = len(datasetNames)
n = spi * lenresults
j = 0
for datasetName,versionno in datasetNames:
if parentId is None:
parentIdent = handler.getParentId(datasetName)
elif type(parentId)==type({}):
parentIdent = parentId[datasetName]
else:
parentIdent = parentId
messaging.info("Publishing: %s"%datasetName)
def deleteDatasetList(datasetNames, Session, gatewayOperation=UNPUBLISH, thredds=True, las=False, deleteInDatabase=False, progressCallback=None,
deleteAll=False, republish=False, reinitThredds=True, restInterface=False, pid_connector=None, project_config_section=None, data_node=None):
"""
Delete or retract a list of datasets:
- Delete the dataset from the gateway.
- Remove the catalogs from the THREDDS catalog (optional).
- Reinitialize the LAS server and THREDDS server.
- Delete the database entry (optional).
if republish is False:
Returns a status dictionary: datasetName => status
else
Returns a tuple (status_dictionary, republishList), where republishList is a list of (dataset_name, version) tuples to be republished.
datasetNames
A list of )dataset_name, version) tuples.
Session
A database Session.
gatewayOperation
An enumeration. If:
- publish.DELETE: Remove all metadata from the gateway database.
- publish.UNPUBLISH: (Default) Remove metadata that allows dataset discovery from the gateway.
- publish.NO_OPERATION: No gateway delete/retract operation is called.
thredds
Boolean flag: if true (the default), delete the associated THREDDS catalog and reinitialize server.
las
Boolean flag: if true (the default), reinitialize server.
deleteInDatabase
Boolean flag: if true (default is False), delete the database entry.
progressCallback
Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported.
deleteAll
Boolean, if True delete all versions of the dataset(s).
republish
Boolean, if True return (statusDictionary, republishList), where republishList is a list of datasets to be republished.
reinitThredds
Boolean flag. If True, create the TDS master catalog and reinitialize the TDS server.
restInterface
Boolean flag. If True, publish datasets with the RESTful publication services.
pid_connector
esgfpid.Connector object to register PIDs
project_config_section
Name of the project config section in esg.ini (for user specific project configs)
data_node
String, the datanode to unpublish (only for unpublication from Solr)
"""
if gatewayOperation == UNINITIALIZED:
raise ESGPublishError("Need to set mandatory --delete|--retract|--skip-index argument!")
if gatewayOperation not in (DELETE, UNPUBLISH, NO_OPERATION):
raise ESGPublishError("Invalid gateway operation: %d"%gatewayOperation)
deleteOnGateway = (gatewayOperation==DELETE)
operation = (gatewayOperation!=NO_OPERATION)
session = Session()
resultDict = {}
config = getConfig()
# Check the dataset names and cache the results for the gateway, thredds, and database phases
nameDict = {}
for datasetName,version in datasetNames:
isDataset, dset, versionObjs, isLatest = datasetOrVersionName(datasetName, version, session, deleteAll=deleteAll, restInterface=restInterface)
if dset is None:
warning("Dataset not found in node database: %s"%datasetName)
nameDict[datasetName] = (isDataset, dset, versionObjs, isLatest)
# Delete the dataset from the gateway.
if operation:
# Create the web service proxy
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL(project_config_section=project_config_section)
servicePort = config.getint('DEFAULT','hessian_service_port')
serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug')
service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug)
else:
service_certs_location = getServiceCertsLoc()
serviceURL = getRestServiceURL(project_config_section=project_config_section)
serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False)
service = RestPublicationService(serviceURL, serviceCertfile, service_certs_location, keyFile=serviceKeyfile, debug=serviceDebug)
for datasetName,version in datasetNames:
if version > -1:
datasetToUnpublish = '%s.v%s' % (datasetName, version)
else:
if service.service_type == 'REST':
error('Cannot unpublish multiple versions using REST. Please specify a single dataset version ("dataset_id#1"). Skipping %s' % datasetName)
continue
datasetToUnpublish = datasetName
isDataset, dset, versionObjs, isLatest = nameDict[datasetName]
try:
eventName, stateName = deleteGatewayDatasetVersion(datasetToUnpublish, gatewayOperation, service, session, dset=dset, data_node=data_node)
except RemoteCallException, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[-2:], '\n')))
continue
info(" Result: %s"%stateName)
resultDict[datasetName] = eventName
# Create the web service proxy
config = getConfig()
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL(project_config_section=project_config_section)
servicePort = config.getint('DEFAULT','hessian_service_port')
serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug')
servicePollingDelay = config.getfloat('DEFAULT','hessian_service_polling_delay')
spi = servicePollingIterations = config.getint('DEFAULT','hessian_service_polling_iterations')
service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug)
else: # REST service
spi = 1
serviceURL = getRestServiceURL(project_config_section=project_config_section)
serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False)
service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug)
results = []
lenresults = len(datasetNames)
n = spi * lenresults
j = 0
for datasetName,versionno in datasetNames:
if parentId is None:
parentIdent = handler.getParentId(datasetName)
elif type(parentId)==type({}):
parentIdent = parentId[datasetName]
else:
parentIdent = parentId
messaging.info("Publishing: %s"%datasetName)
def deleteDatasetList(datasetNames, Session, gatewayOperation=UNPUBLISH, thredds=True, las=False, deleteInDatabase=False, progressCallback=None,
deleteAll=False, republish=False, reinitThredds=True, restInterface=False, pid_connector=None, project_config_section=None, data_node=None):
"""
Delete or retract a list of datasets:
- Delete the dataset from the gateway.
- Remove the catalogs from the THREDDS catalog (optional).
- Reinitialize the LAS server and THREDDS server.
- Delete the database entry (optional).
if republish is False:
Returns a status dictionary: datasetName => status
else
Returns a tuple (status_dictionary, republishList), where republishList is a list of (dataset_name, version) tuples to be republished.
datasetNames
A list of )dataset_name, version) tuples.
Session
A database Session.
gatewayOperation
An enumeration. If:
- publish.DELETE: Remove all metadata from the gateway database.
- publish.UNPUBLISH: (Default) Remove metadata that allows dataset discovery from the gateway.
- publish.NO_OPERATION: No gateway delete/retract operation is called.
thredds
Boolean flag: if true (the default), delete the associated THREDDS catalog and reinitialize server.
las
Boolean flag: if true (the default), reinitialize server.
deleteInDatabase
Boolean flag: if true (default is False), delete the database entry.
progressCallback
Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported.
deleteAll
Boolean, if True delete all versions of the dataset(s).
republish
Boolean, if True return (statusDictionary, republishList), where republishList is a list of datasets to be republished.
reinitThredds
Boolean flag. If True, create the TDS master catalog and reinitialize the TDS server.
restInterface
Boolean flag. If True, publish datasets with the RESTful publication services.
pid_connector
esgfpid.Connector object to register PIDs
project_config_section
Name of the project config section in esg.ini (for user specific project configs)
data_node
String, the datanode to unpublish (only for unpublication from Solr)
"""
if gatewayOperation == UNINITIALIZED:
raise ESGPublishError("Need to set mandatory --delete|--retract|--skip-index argument!")
if gatewayOperation not in (DELETE, UNPUBLISH, NO_OPERATION):
raise ESGPublishError("Invalid gateway operation: %d"%gatewayOperation)
deleteOnGateway = (gatewayOperation==DELETE)
operation = (gatewayOperation!=NO_OPERATION)
session = Session()
resultDict = {}
config = getConfig()
# Check the dataset names and cache the results for the gateway, thredds, and database phases
nameDict = {}
for datasetName,version in datasetNames:
isDataset, dset, versionObjs, isLatest = datasetOrVersionName(datasetName, version, session, deleteAll=deleteAll, restInterface=restInterface)
if dset is None:
warning("Dataset not found in node database: %s"%datasetName)
nameDict[datasetName] = (isDataset, dset, versionObjs, isLatest)
# Delete the dataset from the gateway.
if operation:
# Create the web service proxy
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL(project_config_section=project_config_section)
servicePort = config.getint('DEFAULT','hessian_service_port')
serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug')
service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug)
else:
serviceURL = getRestServiceURL(project_config_section=project_config_section)
serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False)
service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug)
for datasetName,version in datasetNames:
if version > -1:
datasetToUnpublish = '%s.v%s' % (datasetName, version)
else:
datasetToUnpublish = datasetName
isDataset, dset, versionObjs, isLatest = nameDict[datasetName]
try:
eventName, stateName = deleteGatewayDatasetVersion(datasetToUnpublish, gatewayOperation, service, session, dset=dset, data_node=data_node)
except RemoteCallException, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[-2:], '\n')))
continue
info(" Result: %s"%stateName)
resultDict[datasetName] = eventName
def deleteDatasetList(datasetNames, Session, gatewayOperation=UNPUBLISH, thredds=True, las=False, deleteInDatabase=False, progressCallback=None, deleteAll=False, republish=False, restInterface=False):
"""
Delete or retract a list of datasets:
- Delete the dataset from the gateway.
- Remove the catalogs from the THREDDS catalog (optional).
- Reinitialize the LAS server and THREDDS server.
- Delete the database entry (optional).
if republish is False:
Returns a status dictionary: datasetName => status
else
Returns a tuple (status_dictionary, republishList), where republishList is a list of (dataset_name, version) tuples to be republished.
datasetNames
A list of )dataset_name, version) tuples.
Session
A database Session.
gatewayOperation
An enumeration. If:
- publish.DELETE: Remove all metadata from the gateway database.
- publish.UNPUBLISH: (Default) Remove metadata that allows dataset discovery from the gateway.
- publish.NO_OPERATION: No gateway delete/retract operation is called.
thredds
Boolean flag: if true (the default), delete the associated THREDDS catalog and reinitialize server.
las
Boolean flag: if true (the default), reinitialize server.
deleteInDatabase
Boolean flag: if true (default is False), delete the database entry.
progressCallback
Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported.
deleteAll
Boolean, if True delete all versions of the dataset(s).
republish
Boolean, if True return (statusDictionary, republishList), where republishList is a list of datasets to be republished.
restInterface
Boolean flag. If True, publish datasets with the RESTful publication services.
"""
if gatewayOperation not in (DELETE, UNPUBLISH, NO_OPERATION):
raise ESGPublishError("Invalid gateway operation: %d"%gatewayOperation)
deleteOnGateway = (gatewayOperation==DELETE)
operation = (gatewayOperation!=NO_OPERATION)
session = Session()
resultDict = {}
config = getConfig()
# Check the dataset names and cache the results for the gateway, thredds, and database phases
nameDict = {}
for datasetName,version in datasetNames:
isDataset, dset, versionObjs, isLatest = datasetOrVersionName(datasetName, version, session, deleteAll=deleteAll, restInterface=restInterface)
if dset is None:
warning("Dataset not found in node database: %s"%datasetName)
nameDict[datasetName] = (isDataset, dset, versionObjs, isLatest)
# Delete the dataset from the gateway.
if operation:
# Create the web service proxy
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL()
servicePort = config.getint('DEFAULT','hessian_service_port')
serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug')
service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug)
else:
serviceURL = getRestServiceURL()
serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False)
service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug)
for datasetName,version in datasetNames:
isDataset, dset, versionObjs, isLatest = nameDict[datasetName]
if (not DELETE_AT_DATASET_LEVEL) and (dset is not None):
for versionObj in versionObjs:
try:
eventName, stateName = deleteGatewayDatasetVersion(versionObj.name, gatewayOperation, service, session, dset=dset)
except RemoteCallException, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetName, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetName, string.join(fields[-2:], '\n')))
continue
info(" Result: %s"%stateName)
resultDict[datasetName] = eventName
else: # Nothing in the node database, but still try to delete on the gateway
if DELETE_AT_DATASET_LEVEL and (dset is not None) and (not restInterface):
datasetName = dset.name
try:
eventName, stateName = deleteGatewayDatasetVersion(datasetName, gatewayOperation, service, session, dset=dset)
except RemoteCallException, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetName, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetName, string.join(fields[-2:], '\n')))
continue
serviceURL = getHessianServiceURL()
servicePort = config.getint('DEFAULT', 'hessian_service_port')
serviceDebug = config.getboolean('DEFAULT',
'hessian_service_debug')
servicePollingDelay = config.getfloat(
'DEFAULT', 'hessian_service_polling_delay')
spi = servicePollingIterations = config.getint(
'DEFAULT', 'hessian_service_polling_iterations')
service = Hessian(serviceURL,
servicePort,
key_file=serviceKeyfile,
cert_file=serviceCertfile,
debug=serviceDebug)
else: # REST service
spi = 1
serviceURL = getRestServiceURL()
serviceDebug = config.getboolean('DEFAULT',
'rest_service_debug',
default=False)
service = RestPublicationService(serviceURL,
serviceCertfile,
keyFile=serviceKeyfile,
debug=serviceDebug)
results = []
lenresults = len(datasetNames)
n = spi * lenresults
j = 0
for datasetName, versionno in datasetNames:
if parentId is None:
parentIdent = handler.getParentId(datasetName)
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL(project_config_section=project_config_section)
servicePort = config.getint('DEFAULT','hessian_service_port')
serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug')
servicePollingDelay = config.getfloat('DEFAULT','hessian_service_polling_delay')
spi = servicePollingIterations = config.getint('DEFAULT','hessian_service_polling_iterations')
service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug)
else: # REST service
spi = 1
service_certs_location = getServiceCertsLoc()
serviceURL = getRestServiceURL(project_config_section=project_config_section)
serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False)
service = RestPublicationService(serviceURL, serviceCertfile, service_certs_location, keyFile=serviceKeyfile, debug=serviceDebug)
results = []
lenresults = len(datasetNames)
n = spi * lenresults
j = 0
for datasetName,versionno in datasetNames:
if parentId is None:
parentIdent = handler.getParentId(datasetName)
elif type(parentId)==type({}):
parentIdent = parentId[datasetName]
else:
parentIdent = parentId
messaging.info("Publishing: %s"%datasetName)
def deleteDatasetList(datasetNames,
Session,
gatewayOperation=UNPUBLISH,
thredds=True,
las=False,
deleteInDatabase=False,
progressCallback=None,
deleteAll=False,
republish=False,
restInterface=False):
"""
Delete or retract a list of datasets:
- Delete the dataset from the gateway.
- Remove the catalogs from the THREDDS catalog (optional).
- Reinitialize the LAS server and THREDDS server.
- Delete the database entry (optional).
if republish is False:
Returns a status dictionary: datasetName => status
else
Returns a tuple (status_dictionary, republishList), where republishList is a list of (dataset_name, version) tuples to be republished.
datasetNames
A list of )dataset_name, version) tuples.
Session
A database Session.
gatewayOperation
An enumeration. If:
- publish.DELETE: Remove all metadata from the gateway database.
- publish.UNPUBLISH: (Default) Remove metadata that allows dataset discovery from the gateway.
- publish.NO_OPERATION: No gateway delete/retract operation is called.
thredds
Boolean flag: if true (the default), delete the associated THREDDS catalog and reinitialize server.
las
Boolean flag: if true (the default), reinitialize server.
deleteInDatabase
Boolean flag: if true (default is False), delete the database entry.
progressCallback
Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported.
deleteAll
Boolean, if True delete all versions of the dataset(s).
republish
Boolean, if True return (statusDictionary, republishList), where republishList is a list of datasets to be republished.
restInterface
Boolean flag. If True, publish datasets with the RESTful publication services.
"""
if gatewayOperation not in (DELETE, UNPUBLISH, NO_OPERATION):
raise ESGPublishError("Invalid gateway operation: %d" %
gatewayOperation)
deleteOnGateway = (gatewayOperation == DELETE)
operation = (gatewayOperation != NO_OPERATION)
session = Session()
resultDict = {}
config = getConfig()
# Check the dataset names and cache the results for the gateway, thredds, and database phases
nameDict = {}
for datasetName, version in datasetNames:
isDataset, dset, versionObjs, isLatest = datasetOrVersionName(
datasetName,
version,
session,
deleteAll=deleteAll,
restInterface=restInterface)
if dset is None:
warning("Dataset not found in node database: %s" % datasetName)
nameDict[datasetName] = (isDataset, dset, versionObjs, isLatest)
# Delete the dataset from the gateway.
if operation:
# Create the web service proxy
threddsRootURL = config.get('DEFAULT', 'thredds_url')
serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile')
serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile')
if not restInterface:
serviceURL = getHessianServiceURL()
servicePort = config.getint('DEFAULT', 'hessian_service_port')
serviceDebug = config.getboolean('DEFAULT',
'hessian_service_debug')
service = Hessian(serviceURL,
servicePort,
key_file=serviceKeyfile,
cert_file=serviceCertfile,
debug=serviceDebug)
else:
serviceURL = getRestServiceURL()
serviceDebug = config.getboolean('DEFAULT',
'rest_service_debug',
default=False)
service = RestPublicationService(serviceURL,
serviceCertfile,
keyFile=serviceKeyfile,
debug=serviceDebug)
for datasetName, version in datasetNames:
isDataset, dset, versionObjs, isLatest = nameDict[datasetName]
if (not DELETE_AT_DATASET_LEVEL) and (dset is not None):
for versionObj in versionObjs:
try:
eventName, stateName = deleteGatewayDatasetVersion(
versionObj.name,
gatewayOperation,
service,
session,
dset=dset)
except RemoteCallException, e:
fields = ` e `.split('\n')
error(
"Deletion/retraction failed for dataset/version %s with message: %s"
% (datasetName, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = ` e `.split('\n')
error(
"Deletion/retraction failed for dataset/version %s with message: %s"
% (datasetName, string.join(fields[-2:], '\n')))
continue
info(" Result: %s" % stateName)
resultDict[datasetName] = eventName
else: # Nothing in the node database, but still try to delete on the gateway
if DELETE_AT_DATASET_LEVEL and (dset is not None) and (
not restInterface):
datasetName = dset.name
try:
eventName, stateName = deleteGatewayDatasetVersion(
datasetName,
gatewayOperation,
service,
session,
dset=dset)
except RemoteCallException, e:
fields = ` e `.split('\n')
error(
"Deletion/retraction failed for dataset/version %s with message: %s"
% (datasetName, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = ` e `.split('\n')
error(
"Deletion/retraction failed for dataset/version %s with message: %s"
% (datasetName, string.join(fields[-2:], '\n')))
continue