def pollDatasetPublicationStatus(datasetName, Session, service=None): """ Get the current dataset publication status by polling the gateway. Returns the current dataset publication status. datasetNames A list of string dataset names. Session A database Session. service Web service proxy instance. If None, the service is created. """ session = Session() dset = session.query(Dataset).filter_by(name=datasetName).first() if dset is None: messaging.error("Dataset not found: %s" % datasetName) session.close() return PUBLISH_FAILED_EVENT status = dset.get_publication_status() if status != START_PUBLISH_DATASET_EVENT: session.close() return status if service is None: config = getConfig() serviceURL = getHessianServiceURL() servicePort = config.getint('DEFAULT', 'hessian_service_port') serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug') serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile') serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile') service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug) try: statusObj = PublicationStatus(dset.status_id, service) except socket.error, e: raise ESGPublishError( "Socket error: %s\nIs the proxy certificate %s valid?" % ( ` e `, service._cert_file))
messaging.error("Error on LAS reinitialization: %s, new datasets not added."%e) if publish: # Create the web service proxy config = getConfig() threddsRootURL = config.get('DEFAULT', 'thredds_url') serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile') serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile') if not restInterface: serviceURL = getHessianServiceURL(project_config_section=project_config_section) servicePort = config.getint('DEFAULT','hessian_service_port') serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug') servicePollingDelay = config.getfloat('DEFAULT','hessian_service_polling_delay') spi = servicePollingIterations = config.getint('DEFAULT','hessian_service_polling_iterations') service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug) else: # REST service spi = 1 serviceURL = getRestServiceURL(project_config_section=project_config_section) serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False) service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug) results = [] lenresults = len(datasetNames) n = spi * lenresults j = 0 for datasetName,versionno in datasetNames: if parentId is None: parentIdent = handler.getParentId(datasetName) elif type(parentId)==type({}): parentIdent = parentId[datasetName]
def deleteDatasetList(datasetNames, Session, gatewayOperation=UNPUBLISH, thredds=True, las=False, deleteInDatabase=False, progressCallback=None, deleteAll=False, republish=False, reinitThredds=True, restInterface=False, pid_connector=None, project_config_section=None, data_node=None): """ Delete or retract a list of datasets: - Delete the dataset from the gateway. - Remove the catalogs from the THREDDS catalog (optional). - Reinitialize the LAS server and THREDDS server. - Delete the database entry (optional). if republish is False: Returns a status dictionary: datasetName => status else Returns a tuple (status_dictionary, republishList), where republishList is a list of (dataset_name, version) tuples to be republished. datasetNames A list of )dataset_name, version) tuples. Session A database Session. gatewayOperation An enumeration. If: - publish.DELETE: Remove all metadata from the gateway database. - publish.UNPUBLISH: (Default) Remove metadata that allows dataset discovery from the gateway. - publish.NO_OPERATION: No gateway delete/retract operation is called. thredds Boolean flag: if true (the default), delete the associated THREDDS catalog and reinitialize server. las Boolean flag: if true (the default), reinitialize server. deleteInDatabase Boolean flag: if true (default is False), delete the database entry. progressCallback Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported. deleteAll Boolean, if True delete all versions of the dataset(s). republish Boolean, if True return (statusDictionary, republishList), where republishList is a list of datasets to be republished. reinitThredds Boolean flag. If True, create the TDS master catalog and reinitialize the TDS server. restInterface Boolean flag. If True, publish datasets with the RESTful publication services. pid_connector esgfpid.Connector object to register PIDs project_config_section Name of the project config section in esg.ini (for user specific project configs) data_node String, the datanode to unpublish (only for unpublication from Solr) """ if gatewayOperation == UNINITIALIZED: raise ESGPublishError("Need to set mandatory --delete|--retract|--skip-index argument!") if gatewayOperation not in (DELETE, UNPUBLISH, NO_OPERATION): raise ESGPublishError("Invalid gateway operation: %d"%gatewayOperation) deleteOnGateway = (gatewayOperation==DELETE) operation = (gatewayOperation!=NO_OPERATION) session = Session() resultDict = {} config = getConfig() # Check the dataset names and cache the results for the gateway, thredds, and database phases nameDict = {} for datasetName,version in datasetNames: isDataset, dset, versionObjs, isLatest = datasetOrVersionName(datasetName, version, session, deleteAll=deleteAll, restInterface=restInterface) if dset is None: warning("Dataset not found in node database: %s"%datasetName) nameDict[datasetName] = (isDataset, dset, versionObjs, isLatest) # Delete the dataset from the gateway. if operation: # Create the web service proxy threddsRootURL = config.get('DEFAULT', 'thredds_url') serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile') serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile') if not restInterface: serviceURL = getHessianServiceURL(project_config_section=project_config_section) servicePort = config.getint('DEFAULT','hessian_service_port') serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug') service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug) else: serviceURL = getRestServiceURL(project_config_section=project_config_section) serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False) service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug) for datasetName,version in datasetNames: if version > -1: datasetToUnpublish = '%s.v%s' % (datasetName, version) else: datasetToUnpublish = datasetName isDataset, dset, versionObjs, isLatest = nameDict[datasetName] try: eventName, stateName = deleteGatewayDatasetVersion(datasetToUnpublish, gatewayOperation, service, session, dset=dset, data_node=data_node) except RemoteCallException, e: fields = `e`.split('\n') error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[0:2], '\n'))) continue except ESGPublishError, e: fields = `e`.split('\n') error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[-2:], '\n'))) continue info(" Result: %s"%stateName) resultDict[datasetName] = eventName
def deleteDatasetList(datasetNames, Session, gatewayOperation=UNPUBLISH, thredds=True, las=False, deleteInDatabase=False, progressCallback=None, deleteAll=False, republish=False, restInterface=False): """ Delete or retract a list of datasets: - Delete the dataset from the gateway. - Remove the catalogs from the THREDDS catalog (optional). - Reinitialize the LAS server and THREDDS server. - Delete the database entry (optional). if republish is False: Returns a status dictionary: datasetName => status else Returns a tuple (status_dictionary, republishList), where republishList is a list of (dataset_name, version) tuples to be republished. datasetNames A list of )dataset_name, version) tuples. Session A database Session. gatewayOperation An enumeration. If: - publish.DELETE: Remove all metadata from the gateway database. - publish.UNPUBLISH: (Default) Remove metadata that allows dataset discovery from the gateway. - publish.NO_OPERATION: No gateway delete/retract operation is called. thredds Boolean flag: if true (the default), delete the associated THREDDS catalog and reinitialize server. las Boolean flag: if true (the default), reinitialize server. deleteInDatabase Boolean flag: if true (default is False), delete the database entry. progressCallback Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported. deleteAll Boolean, if True delete all versions of the dataset(s). republish Boolean, if True return (statusDictionary, republishList), where republishList is a list of datasets to be republished. restInterface Boolean flag. If True, publish datasets with the RESTful publication services. """ if gatewayOperation not in (DELETE, UNPUBLISH, NO_OPERATION): raise ESGPublishError("Invalid gateway operation: %d" % gatewayOperation) deleteOnGateway = (gatewayOperation == DELETE) operation = (gatewayOperation != NO_OPERATION) session = Session() resultDict = {} config = getConfig() # Check the dataset names and cache the results for the gateway, thredds, and database phases nameDict = {} for datasetName, version in datasetNames: isDataset, dset, versionObjs, isLatest = datasetOrVersionName( datasetName, version, session, deleteAll=deleteAll, restInterface=restInterface) if dset is None: warning("Dataset not found in node database: %s" % datasetName) nameDict[datasetName] = (isDataset, dset, versionObjs, isLatest) # Delete the dataset from the gateway. if operation: # Create the web service proxy threddsRootURL = config.get('DEFAULT', 'thredds_url') serviceCertfile = config.get('DEFAULT', 'hessian_service_certfile') serviceKeyfile = config.get('DEFAULT', 'hessian_service_keyfile') if not restInterface: serviceURL = getHessianServiceURL() servicePort = config.getint('DEFAULT', 'hessian_service_port') serviceDebug = config.getboolean('DEFAULT', 'hessian_service_debug') service = Hessian(serviceURL, servicePort, key_file=serviceKeyfile, cert_file=serviceCertfile, debug=serviceDebug) else: serviceURL = getRestServiceURL() serviceDebug = config.getboolean('DEFAULT', 'rest_service_debug', default=False) service = RestPublicationService(serviceURL, serviceCertfile, keyFile=serviceKeyfile, debug=serviceDebug) for datasetName, version in datasetNames: isDataset, dset, versionObjs, isLatest = nameDict[datasetName] if (not DELETE_AT_DATASET_LEVEL) and (dset is not None): for versionObj in versionObjs: try: eventName, stateName = deleteGatewayDatasetVersion( versionObj.name, gatewayOperation, service, session, dset=dset) except RemoteCallException, e: fields = ` e `.split('\n') error( "Deletion/retraction failed for dataset/version %s with message: %s" % (datasetName, string.join(fields[0:2], '\n'))) continue except ESGPublishError, e: fields = ` e `.split('\n') error( "Deletion/retraction failed for dataset/version %s with message: %s" % (datasetName, string.join(fields[-2:], '\n'))) continue info(" Result: %s" % stateName) resultDict[datasetName] = eventName else: # Nothing in the node database, but still try to delete on the gateway if DELETE_AT_DATASET_LEVEL and (dset is not None) and ( not restInterface): datasetName = dset.name try: eventName, stateName = deleteGatewayDatasetVersion( datasetName, gatewayOperation, service, session, dset=dset) except RemoteCallException, e: fields = ` e `.split('\n') error( "Deletion/retraction failed for dataset/version %s with message: %s" % (datasetName, string.join(fields[0:2], '\n'))) continue except ESGPublishError, e: fields = ` e `.split('\n') error( "Deletion/retraction failed for dataset/version %s with message: %s" % (datasetName, string.join(fields[-2:], '\n'))) continue