def __init__( self, name, **args ):
self.getIntracom()
self.cacheManager = CacheManager( name, comm=self.intracom )
self.collectionManager = getCollectionManger( **args )
self.persist_queue = Queue.Queue()
self.persistenceThread = None
enable_background_persist = args.get( 'background_persist', True )
if enable_background_persist:
self.persistenceThread = PersistenceThread( args=(self.persist_queue,) )
self.persistenceThread.start()
class DataManager:
def __init__( self, name, **args ):
self.getIntracom()
self.cacheManager = CacheManager( name, comm=self.intracom )
self.collectionManager = getCollectionManger( **args )
self.persist_queue = Queue.Queue()
self.persistenceThread = None
enable_background_persist = args.get( 'background_persist', True )
if enable_background_persist:
self.persistenceThread = PersistenceThread( args=(self.persist_queue,) )
self.persistenceThread.start()
def getIntracom(self):
from engines import engineRegistry
from modules.configuration import CDAS_COMPUTE_ENGINE
self.intracom = engineRegistry.getWorkerIntracom( CDAS_COMPUTE_ENGINE + "Engine" )
def transferDomain( self, source, destination, domain_spec, subregion_spec ):
subregion = Region(subregion_spec) if (subregion_spec is not None) else None
if source == self.cacheManager.name:
transfer_data = self.cacheManager.sendData( destination, domain_spec, subregion )
elif destination == self.cacheManager.name:
transfer_data = self.cacheManager.receiveData( source, domain_spec, subregion )
else: transfer_data = None
return transfer_data
def close(self):
self.collectionManager.close()
def getName(self):
return self.cacheManager.name
def persist( self, **args ):
self.cacheManager.persist( **args )
def persistStats( self, **args ):
self.cacheManager.persistStats( **args )
def uncache( self, data, region ):
self.cacheManager.uncache( data, region )
def stats( self, **args ):
return self.cacheManager.stats( **args )
def load_variable_region( self, dataset, name, region ):
from decomposition.strategies import decimationManager
cdms2_cache_args = None
rv = None
try:
t0 = time.time()
wid = self.cacheManager.name
variable = dataset[name]
load_region = region # decimationManager.getReducedRegion( region, axes=variable.getAxisList() )
cdms2_cache_args = load_region.toCDMS()
wpsLog.debug( "\n\n LLLLLLLOAD DataSet<%s:%s> %x:%x, status = '%s', var=%s, args=%s \n\n" % ( dataset.id, wid, id(dataset), os.getpid(), dataset._status_, name, str(cdms2_cache_args) ) )
dset = variable( **cdms2_cache_args )
rv = numpy.ma.fix_invalid( dset )
t1 = time.time()
wpsLog.debug( " $$$ Variable '%s' %s loaded from Dataset<%s:%s> --> TIME: %.2f " % ( name, str(rv.shape), dataset.id, wid, (t1-t0) ) )
except Exception, err:
wpsLog.error( " ERROR loading Variable '%s' from dataset %s --> args: %s\n --- %s ----\n%s " % ( name, str(dataset), str(cdms2_cache_args), str(err), traceback.format_exc() ) )
return rv