def publishDatasetList(datasetNames, Session, parentId=None, handlerDictionary=None, publish=True, thredds=True, las=False, progressCallback=None, service=None, perVariable=None, threddsCatalogDictionary=None, reinitThredds=None, readFromCatalog=False, restInterface=False, schema=None):
"""
Publish a list of datasets:
- For each dataset, write a THREDDS catalog.
- Add the new catalogs to the THREDDS catalog tree and reinitilize the THREDDS server.
- Reinitialize the LAS server.
- Publish each dataset to the gateway.
Returns a dictionary: (datasetName, version) => status
datasetNames
A list of (string_dataset_name, version) tuples.
Session
A database Session.
parentId
The string (or dictionary) persistent identifier of the parent of the datasets. If None (the default),
the parent id for each dataset is generated from ``handler.getParentId()``. If a dictionary, each
dataset name is used as a key to lookup the respective parent id. If a string, the parent id is
set to the string for all datasets being published. This function
can be overridden in the project handler to implement a project-specific dataset hierarchy.
handlerDictionary
A dictionary mapping dataset_name => handler.
publish
Boolean flag: if true (the default), contact the gateway to publish this dataset.
thredds
Boolean flag: if true (the default), write the associated THREDDS catalog.
las
Boolean flag: if true (the default), write the associated LAS catalog.
progressCallback
Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported.
service
String service name. If omitted, the first online/offline service in the configuration is used.
perVariable
Boolean, overrides ``variable_per_file`` config option.
threddsCatalogDictionary
If not None, just generate catalogs in strings, not the THREDDS directories, and set
threddsCatalogDictionary[datasetId] = string_catalog
reinitThredds
Boolean flag. If True, create the TDS master catalog and reinitialize the TDS server.
If None, defaults to value of thredds option.
readFromCatalog
Boolean flag. If True, read the TDS catalog definitions from threddsCatalogDictionary.
threddsCatalogDictionary must also be set.
restInterface
Boolean flag. If True, publish datasets with the RESTful publication services.
schema
(Optional) String name of the schema to validate against, for RESTful publication calls.
"""
session = Session()
resultDict = {}
if readFromCatalog and threddsCatalogDictionary is None:
raise ESGPublishError("Must set THREDDS catalog dictionary when readFromCatalog is True.")
# Get handlers for each dataset
if handlerDictionary is None:
handlers = {}
for datasetName,versionno in datasetNames:
dset = session.query(Dataset).filter_by(name=datasetName).first()
if dset is None:
raise ESGPublishError("Dataset not found: %s"%datasetName)
handler = getHandlerByName(dset.project, None, Session)
handlers[datasetName] = handler
else:
handlers = handlerDictionary
# reinitThredds defaults to the value of thredds option
if reinitThredds is None:
reinitThredds = thredds
if thredds:
for datasetName,versionno in datasetNames:
dset = session.query(Dataset).filter_by(name=datasetName).first()
# If the dataset version is not the latest, publish as a per-time dataset without aggregation,
# since the dataset variables only relate to the latest dataset version
latestVersion = dset.getVersion()
if versionno==-1:
versionno=latestVersion
if versionno!=latestVersion:
if perVariable:
messaging.info("Generating THREDDS catalog in per-time format, since version %d is not the latest version (%d)"%(versionno,latestVersion))
perVariable = False
handler = handlers[datasetName]
# If threddsCatalogDictionary is not set, create the TDS catalog in the TDS content directory ...
if threddsCatalogDictionary is None:
threddsOutputPath = generateThreddsOutputPath(datasetName, versionno, Session, handler)
threddsOutput = open(threddsOutputPath, "w")
generateThredds(datasetName, Session, threddsOutput, handler, service=service, perVariable=perVariable, versionNumber=versionno)
threddsOutput.close()
try:
os.chmod(threddsOutputPath, 0664)
except:
pass
# ... else if threddsCatalogDictionary is the catalog source:
elif readFromCatalog:
catalogString = threddsCatalogDictionary[(datasetName,versionno)]
threddsOutputPath = generateThreddsOutputPath(datasetName, versionno, Session, handler)
threddsOutput = open(threddsOutputPath, "w")
messaging.info("Writing THREDDS catalog %s"%threddsOutputPath)
threddsOutput.write(catalogString)
threddsOutput.close()
try:
os.chmod(threddsOutputPath, 0664)
except:
pass
# ... otherwise write the catalog in a 'string file'
else:
threddsOutputPath = generateThreddsOutputPath(datasetName, versionno, Session, handler) # Creates catalog entry
threddsOutput = cStringIO.StringIO()
generateThredds(datasetName, Session, threddsOutput, handler, service=service, perVariable=perVariable, versionNumber=versionno)
threddsCatalogDictionary[(datasetName,versionno)] = threddsOutput.getvalue()
threddsOutput.close()
if reinitThredds:
updateThreddsMasterCatalog(Session)
result = reinitializeThredds()
if las:
try:
result = reinitializeLAS()
except Exception, e:
messaging.error("Error on LAS reinitialization: %s, new datasets not added."%e)
try:
eventName, stateName = deleteGatewayDatasetVersion(datasetToUnpublish, gatewayOperation, service, session, dset=dset, data_node=data_node)
except RemoteCallException, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[-2:], '\n')))
continue
info(" Result: %s"%stateName)
resultDict[datasetName] = eventName
# Reinitialize the LAS server.
if las:
result = reinitializeLAS()
# Remove the catalogs from the THREDDS catalog (optional),
# and reinitialize the THREDDS server.
if thredds:
threddsRoot = config.get('DEFAULT', 'thredds_root')
for datasetName,version in datasetNames:
isDataset, dset, versionObjs, isLatest = nameDict[datasetName]
if dset is None:
continue
for versionObj in versionObjs:
# send unpublication info to handle server
if pid_connector:
pid_connector.unpublish_one_version(drs_id=datasetName, version_number=versionObj.version)
catalog = session.query(Catalog).filter_by(dataset_name=dset.name, version=versionObj.version).first()
if catalog is not None:
def publishDatasetList(datasetNames, Session, parentId=None, handlerDictionary=None, publish=True, thredds=True, las=False, progressCallback=None,
service=None, perVariable=None, threddsCatalogDictionary=None, reinitThredds=None, readFromCatalog=False, restInterface=False,
schema=None, pid_connector=None, project_config_section=None):
"""
Publish a list of datasets:
- For each dataset, write a THREDDS catalog.
- Add the new catalogs to the THREDDS catalog tree and reinitilize the THREDDS server.
- Reinitialize the LAS server.
- Publish each dataset to the gateway.
Returns a dictionary: (datasetName, version) => status
datasetNames
A list of (string_dataset_name, version) tuples.
Session
A database Session.
parentId
The string (or dictionary) persistent identifier of the parent of the datasets. If None (the default),
the parent id for each dataset is generated from ``handler.getParentId()``. If a dictionary, each
dataset name is used as a key to lookup the respective parent id. If a string, the parent id is
set to the string for all datasets being published. This function
can be overridden in the project handler to implement a project-specific dataset hierarchy.
handlerDictionary
A dictionary mapping dataset_name => handler.
publish
Boolean flag: if true (the default), contact the gateway to publish this dataset.
thredds
Boolean flag: if true (the default), write the associated THREDDS catalog.
las
Boolean flag: if true (the default), write the associated LAS catalog.
progressCallback
Tuple (callback, initial, final) where ``callback`` is a function of the form ``callback(progress)``, ``initial`` is the initial value reported, ``final`` is the final value reported.
service
String service name. If omitted, the first online/offline service in the configuration is used.
perVariable
Boolean, overrides ``variable_per_file`` config option.
threddsCatalogDictionary
If not None, just generate catalogs in strings, not the THREDDS directories, and set
threddsCatalogDictionary[datasetId] = string_catalog
reinitThredds
Boolean flag. If True, create the TDS master catalog and reinitialize the TDS server.
If None, defaults to value of thredds option.
readFromCatalog
Boolean flag. If True, read the TDS catalog definitions from threddsCatalogDictionary.
threddsCatalogDictionary must also be set.
restInterface
Boolean flag. If True, publish datasets with the RESTful publication services.
schema
(Optional) String name of the schema to validate against, for RESTful publication calls.
pid_connector
esgfpid.Connector object to register PIDs
project_config_section
Name of the project config section in esg.ini (for user specific project configs)
"""
session = Session()
resultDict = {}
if readFromCatalog and threddsCatalogDictionary is None:
raise ESGPublishError("Must set THREDDS catalog dictionary when readFromCatalog is True.")
# Get handlers for each dataset
if handlerDictionary is None:
handlers = {}
for datasetName,versionno in datasetNames:
dset = session.query(Dataset).filter_by(name=datasetName).first()
if dset is None:
raise ESGPublishError("Dataset not found: %s"%datasetName)
handler = getHandlerByName(dset.project, None, Session)
handlers[datasetName] = handler
else:
handlers = handlerDictionary
# reinitThredds defaults to the value of thredds option
if reinitThredds is None:
reinitThredds = thredds
if thredds:
for datasetName,versionno in datasetNames:
dset = session.query(Dataset).filter_by(name=datasetName).first()
# If the dataset version is not the latest, publish as a per-time dataset without aggregation,
# since the dataset variables only relate to the latest dataset version
latestVersion = dset.getVersion()
if versionno==-1:
versionno=latestVersion
if versionno!=latestVersion:
if perVariable:
messaging.info("Generating THREDDS catalog in per-time format, since version %d is not the latest version (%d)"%(versionno,latestVersion))
perVariable = False
handler = handlers[datasetName]
# If threddsCatalogDictionary is not set, create the TDS catalog in the TDS content directory ...
if threddsCatalogDictionary is None:
threddsOutputPath = generateThreddsOutputPath(datasetName, versionno, Session, handler)
threddsOutput = open(threddsOutputPath, "w")
generateThredds(datasetName, Session, threddsOutput, handler, service=service, perVariable=perVariable, versionNumber=versionno,
pid_connector=pid_connector)
threddsOutput.close()
try:
os.chmod(threddsOutputPath, 0664)
except:
pass
# ... else if threddsCatalogDictionary is the catalog source:
elif readFromCatalog:
catalogString = threddsCatalogDictionary[(datasetName,versionno)]
threddsOutputPath = generateThreddsOutputPath(datasetName, versionno, Session, handler)
threddsOutput = open(threddsOutputPath, "w")
messaging.info("Writing THREDDS catalog %s"%threddsOutputPath)
threddsOutput.write(catalogString)
threddsOutput.close()
try:
os.chmod(threddsOutputPath, 0664)
except:
pass
# ... otherwise write the catalog in a 'string file'
else:
threddsOutputPath = generateThreddsOutputPath(datasetName, versionno, Session, handler) # Creates catalog entry
threddsOutput = cStringIO.StringIO()
generateThredds(datasetName, Session, threddsOutput, handler, service=service, perVariable=perVariable, versionNumber=versionno,
pid_connector=pid_connector)
threddsCatalogDictionary[(datasetName,versionno)] = threddsOutput.getvalue()
threddsOutput.close()
if reinitThredds:
updateThreddsMasterCatalog(Session)
result = reinitializeThredds()
if las:
try:
result = reinitializeLAS()
except Exception, e:
messaging.error("Error on LAS reinitialization: %s, new datasets not added."%e)
try:
eventName, stateName = deleteGatewayDatasetVersion(datasetToUnpublish, gatewayOperation, service, session, dset=dset, data_node=data_node)
except RemoteCallException, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[0:2], '\n')))
continue
except ESGPublishError, e:
fields = `e`.split('\n')
error("Deletion/retraction failed for dataset/version %s with message: %s"%(datasetToUnpublish, string.join(fields[-2:], '\n')))
continue
info(" Result: %s"%stateName)
resultDict[datasetName] = eventName
# Reinitialize the LAS server.
if las:
result = reinitializeLAS()
# Remove the catalogs from the THREDDS catalog (optional),
# and reinitialize the THREDDS server.
if thredds:
threddsRoot = config.get('DEFAULT', 'thredds_root')
for datasetName,version in datasetNames:
isDataset, dset, versionObjs, isLatest = nameDict[datasetName]
if dset is None:
continue
for versionObj in versionObjs:
# send unpublication info to handle server
if pid_connector:
pid_connector.unpublish_one_version(drs_id=datasetName, version_number=versionObj.version)
catalog = session.query(Catalog).filter_by(dataset_name=dset.name, version=versionObj.version).first()
if catalog is not None: