def __init__(self,
client_address: str = None,
request_port: int = None,
response_port: int = None):
super(EDASapp, self).__init__(
get_or_else(client_address, EdasEnv.get("wps.server.address",
"*")),
get_or_else(request_port, EdasEnv.get("request.port", 4556)),
get_or_else(response_port, EdasEnv.get("response.port", 4557)))
self.process = "edas"
self.processManager = None
atexit.register(self.term, "ShutdownHook Called")
self.logger.info("STARTUP CLUSTER")
self.processManager = ProcessManager.initManager(EdasEnv.parms)
self.scheduler_info = self.processManager.client.scheduler_info()
workers: Dict = self.scheduler_info.pop("workers")
self.logger.info(" @@@@@@@ SCHEDULER INFO: " +
str(self.scheduler_info))
self.logger.info(f" N Workers: {len(workers)} ")
for addr, specs in workers.items():
self.logger.info(f" -----> Worker {addr}: {specs}")
log_metrics = EdasEnv.parms.get("log.cwt.metrics", True)
if log_metrics:
self.metricsThread = Thread(target=self.trackCwtMetrics)
self.metricsThread.start()
def getSession(self, dataSource: DataSource) -> Session:
session: Session = None
if dataSource.auth == "esgf":
from pydap.cas.esgf import setup_session
openid = EdasEnv.get("esgf.openid", "")
password = EdasEnv.get("esgf.password", "")
username = EdasEnv.get("esgf.username", openid.split("/")[-1])
session = setup_session(openid,
password,
username,
check_url=dataSource.address)
elif dataSource.auth == "urs":
from pydap.cas.urs import setup_session
username = EdasEnv.get("urs.username", "")
password = EdasEnv.get("urs.password", "")
session = setup_session(username,
password,
check_url=dataSource.address)
elif dataSource.auth == "cookie":
from pydap.cas.get_cookies import setup_session
username = EdasEnv.get("auth.username", "")
password = EdasEnv.get("auth.password", "")
auth_url = EdasEnv.get("auth.url", "")
session = setup_session(auth_url, username, password)
elif dataSource.auth is not None:
raise Exception("Unknown authentication method: " +
dataSource.auth)
return session
def __init__(self,
nthreads=0,
nprocs=1,
nohost=False,
remote_python=None,
memory_limit=None,
worker_port=None,
nanny_port=None):
Thread.__init__(self)
self.logger = EDASLogger.getLogger()
self.nthreads = nthreads
self.nprocs = nprocs
self.worker_addrs = self.getHosts()
self.ssh_username = os.environ.get('USER', None)
self.ssh_port = 22
self.ssh_private_key = get_private_key()
self.scheduler_addr = getHost()
self.scheduler_port = int(EdasEnv.get("scheduler.port", 8786))
self.logdir = os.path.expanduser("~/.edas/logs")
self.active = False
self.nohost = nohost
self.remote_python = remote_python
self.memory_limit = memory_limit
self.worker_port = worker_port
self.nanny_port = nanny_port
# Keep track of all running threads
self.threads = []
def buildWorkflow(self, request: TaskRequest, node: WorkflowNode, inputs: EDASDatasetCollection ) -> EDASDatasetCollection:
snode: SourceNode = node
results = EDASDatasetCollection( "InputKernel.build-" + node.name )
t0 = time.time()
dset = self.getCachedDataset( snode )
if dset is not None:
self.importToDatasetCollection(results, request, snode, dset.xr )
self.logger.info( "Access input data from cache: " + dset.id )
else:
dataSource: DataSource = snode.varSource.dataSource
if dataSource.type == SourceType.collection:
from edas.collection.agg import Axis as AggAxis, File as AggFile
collection = Collection.new( dataSource.address )
self.logger.info("Input collection: " + dataSource.address )
aggs = collection.sortVarsByAgg( snode.varSource.vids )
domain = request.operationManager.domains.getDomain( snode.domain )
if domain is not None:
timeBounds = domain.findAxisBounds(Axis.T)
startDate = None if (domain is None or timeBounds is None) else TimeConversions.parseDate(timeBounds.start)
endDate = None if (domain is None or timeBounds is None) else TimeConversions.parseDate(timeBounds.end)
else: startDate = endDate = None
for ( aggId, vars ) in aggs.items():
use_chunks = True
pathList = collection.pathList(aggId) if startDate is None else collection.periodPathList(aggId,startDate,endDate)
assert len(pathList) > 0, f"No files found in aggregation {aggId} for date range {startDate} - {endDate} "
nFiles = len(pathList)
if use_chunks:
nReadPartitions = int( EdasEnv.get( "mfdataset.npartitions", 250 ) )
agg = collection.getAggregation(aggId)
nchunks, fileSize = agg.getChunkSize( nReadPartitions, nFiles )
chunk_kwargs = {} if nchunks is None else dict(chunks={"time": nchunks})
self.logger.info( f"Open mfdataset: vars={vars}, NFILES={nFiles}, FileSize={fileSize}, FILES[0]={pathList[0]}, chunk_kwargs={chunk_kwargs}, startDate={startDate}, endDate={endDate}, domain={domain}" )
else:
chunk_kwargs = {}
self.logger.info( f"Open mfdataset: vars={vars}, NFILES={nFiles}, FILES[0]={pathList[0]}" )
dset = xr.open_mfdataset( pathList, engine='netcdf4', data_vars=vars, parallel=True, **chunk_kwargs )
self.logger.info(f"Import to collection")
self.importToDatasetCollection( results, request, snode, dset )
self.logger.info(f"Collection import complete.")
elif dataSource.type == SourceType.file:
self.logger.info( "Reading data from address: " + dataSource.address )
files = glob.glob( dataSource.address )
parallel = len(files) > 1
assert len(files) > 0, f"No files matching path {dataSource.address}"
dset = xr.open_mfdataset(dataSource.address, engine='netcdf4', data_vars=snode.varSource.ids, parallel=parallel )
self.importToDatasetCollection(results, request, snode, dset)
elif dataSource.type == SourceType.archive:
self.logger.info( "Reading data from archive: " + dataSource.address )
dataPath = request.archivePath( dataSource.address )
dset = xr.open_mfdataset( [dataPath] )
self.importToDatasetCollection(results, request, snode, dset)
elif dataSource.type == SourceType.dap:
nchunks = request.runargs.get( "ncores", 8 )
self.logger.info( f" --------------->>> Reading data from address: {dataSource.address}, nchunks = {nchunks}" )
# dset = xr.open_mfdataset( [dataSource.address], engine="netcdf4", data_vars=snode.varSource.ids, chunks={"time":nchunks} )
dset = xr.open_dataset( dataSource.address, engine="netcdf4", chunks={"time":nchunks} )
self.importToDatasetCollection( results, request, snode, dset )
self.logger.info( f"Access input data source {dataSource.address}, time = {time.time() - t0} sec" )
self.logger.info( "@L: LOCATION=> host: {}, thread: {}, proc: {}".format( socket.gethostname(), threading.get_ident(), os.getpid() ) )
return results
def __init__(self):
Cluster.__init__(self)
self.logger = EDASLogger.getLogger()
self.scheduler_host = getHost()
self.scheduler_port = int(EdasEnv.get("scheduler.port", 8786))
self.schedulerProcess = self.startup_scheduler()
time.sleep(14)
self.clusterThread = self.startup_cluster()
def getHosts(self):
hostfile = EdasEnv.get("hostfile.path",
os.path.expanduser("~/.edas/conf/hosts"))
assert os.path.isfile(
hostfile
), "Error, the EDAS hosts file '{}' does not exist. Copy edas/resourses/hosts.template to '{}' and edit.".format(
hostfile, hostfile)
with open(hostfile) as f:
return f.read().split()
def validate(cls, _address: str, stype: SourceType = SourceType.uri):
allowed_sources = [
r.strip() for r in EdasEnv.get("sources.allowed",
"collection,https").split(",")
]
toks = _address.split(":")
scheme = toks[0].lower()
if (stype.name.lower() == "uri") and (scheme in allowed_sources):
if scheme == "https":
trusted_servers = [
r.strip()
for r in EdasEnv.get("trusted.dap.servers", "").split(",")
]
for trusted_server in trusted_servers:
if trusted_server in _address: return scheme, toks[1]
raise Exception(
f"Attempt to access untrusted dap server: {_address}\n\t Trusted servers: {trusted_servers}\n\t Use parameter 'trusted.dap.servers' in app.conf to list trusted addresses, e.g. 'trusted.dap.servers=https://aims3.llnl.gov/thredds/dodsC/'"
)
else:
return scheme, toks[1]
else:
raise Exception("Unallowed scheme '{}' in url: {}".format(
scheme, _address))
def startup_scheduler(self):
os.environ["PATH"] = ":".join([self.EDAS_BIN_DIR, os.environ["PATH"]])
if not EdasEnv.getBool("edas.manage.scheduler", True): return None
# os.environ["PKEY_OPTS"] = "--ssh-private-key=" + get_private_key()
os.environ["PATH"] = ":".join([self.EDAS_BIN_DIR, os.environ["PATH"]])
bokeh_port = int(EdasEnv.get("dashboard.port", 8787))
self.logger.info(
"Starting up scheduler using script {} with host {} and port {}".
format(self.SCHEDULER_SCRIPT, self.scheduler_host,
self.scheduler_port))
args = [
sys.executable, self.SCHEDULER_SCRIPT, "--host",
self.scheduler_host, "--port",
str(self.scheduler_port), "--bokeh-port",
str(bokeh_port)
]
return subprocess.Popen(args, stderr=subprocess.PIPE)
def __init__(self):
self.arrayCache: Dict[str, EDASArray] = OrderedDict()
self.maxSize = SizeParser.parse(EdasEnv.get("cache.size.max", "500M"))
self.currentSize = 0