def launch_python_post():
curDir = os.path.dirname(os.path.abspath(__file__))
logger = PyPostTools.pyPostLogger()
logger.write("Initializing WRF Python Post-Processing Program")
#Step 1: Load program settings
logger.write(" 1. Application Initalization")
logger.write(" - Loading control file, python_post_control.txt")
_pySet = PyPostSettings.PyPostSettings()
logger.write(" - Success!")
logger.write(" - Testing Environmental Variables")
try:
dask_nodes = os.environ["PYTHON_POST_NODES"]
dask_threads = os.environ["PYTHON_POST_THREADS"]
postDir = os.environ["PYTHON_POST_DIR"]
targetDir = os.environ["PYTHON_POST_TARG_DIR"]
except KeyError:
logger.write("***FAIL*** KeyError encountered while trying to access important environmental variables, abort.")
sys.exit("")
logger.write(" - Success!")
logger.write(" - Initializing Dask (" + str(dask_nodes) + " Nodes Requested), Collecting routines needed")
_routines = Routines.Routines()
logger.write(" - Async IO Loop initialized...")
def f(scheduler_port):
async def g(port):
s = Scheduler(port=scheduler_port)
await s
await s.finished()
asyncio.get_event_loop().run_until_complete(g(scheduler_port))
# Starts the scheduler in its own process - needed as otherwise it will
# occupy the program and make it do an infinite loop
process = Process(target=f, args=(scheduler_port,))
process.start()
logger.write(" - Dask Scheduler initialized (Port " + str(scheduler_port) + ")...")
try:
dask_client = Client("tcp://" + socket.gethostname() + ":" + str(scheduler_port), timeout=30)
except OSError:
logger.write(" <-> Dask Client could not be created, timeout error.")
process.terminate()
sys.exit()
logger.write(" - Dask Client initialized...")
logger.write(" - Writing Dask Worker Job Files...")
with PyPostTools.cd(targetDir):
writeFile = PyPostTools.write_job_file(socket.gethostname(), scheduler_port, project_name="climate_severe", queue="debug-cache-quad", nodes=dask_nodes, wall_time=60, nProcs=1)
if(writeFile == False):
dask_client.close()
logger.write(" - Failed to write job file, are you missing an important parameter?")
sys.exit("")
return
else:
logger.write(" - Dask Worker Job File Written, Submitting to Queue.")
PyPostTools.popen("chmod +x dask-worker.job")
PyPostTools.popen("qsub dask-worker.job")
# Wait here for workers.
logger.write(" -> Worker Job submitted to queue, waiting for workers...")
while len(dask_client.scheduler_info()['workers']) < int(dask_nodes):
time.sleep(2)
logger.write(" -> Workers are now connected.")
logger.write(" - Success!")
logger.write(" 1. Done.")
logger.write(" 2. Start Post-Processing Calculations")
start_calculations(dask_client, _routines, dask_threads, process)
logger.write(" 2. Done.")
logger.write(" 3. Generating Figures")
logger.write(" - Collecting files from target directory (" + targetDir + ").")
fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*"))
logger.write(" - " + str(len(fList3)) + " files have been found.")
logger.write(" -> Pushing run_plotting_routines() to dask.")
fullDict = _pySet.get_full_dict()
plotting_future = start_plotting(dask_client, fullDict, dask_threads, process)
wait(plotting_future)
result_plot = dask_client.gather(plotting_future)[0]
if(result_plot != 0):
logger.write("***FAIL*** An error occured in plotting method, check worker logs for more info.")
logger.close()
sys.exit("")
logger.write(" 3. Done.")
logger.write(" 4. Final Steps")
logger.write(" 4. Done, Closing Dask Client.")
# Close the client object
dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'], close=True)
dask_client.close()
logger.write("All Steps Completed.")
logger.write("***SUCCESS*** Program execution complete.")
logger.close()
del dask_client
process.terminate()
def launch_python_post():
curDir = os.path.dirname(os.path.abspath(__file__))
logger = PyPostTools.pyPostLogger()
logger.write("Initializing WRF Python Post-Processing Program")
#Step 1: Load program settings
logger.write(" 1. Application Initalization")
logger.write(" - Loading control file, python_post_control.txt")
_pySet = PyPostSettings.PyPostSettings()
logger.write(" - Success!")
logger.write(" - Testing Environmental Variables")
try:
dask_nodes = os.environ["PYTHON_POST_NODES"]
dask_threads = os.environ["PYTHON_POST_THREADS"]
postDir = os.environ["PYTHON_POST_DIR"]
targetDir = os.environ["PYTHON_POST_TARG_DIR"]
except KeyError:
logger.write(
"***FAIL*** KeyError encountered while trying to access important environmental variables, abort."
)
sys.exit("")
logger.write(" - Success!")
logger.write(" - Initializing Dask (" + str(dask_nodes) +
" Nodes Requested), Collecting routines needed")
_routines = Routines.Routines()
# Start Dask Tasks
#cLoop = IOLoop.current()
#t = Thread(target = cLoop.start, daemon = True)
#t.start()
logger.write(" - Async IO Loop initialized...")
async def f(port):
s = Scheduler(port=scheduler_port)
s = await s
await s.finished()
return 1
asyncio.gather(f(scheduler_port))
#asyncio.get_event_loop().run_until_complete(f(scheduler_port))
logger.write(" - Dask Scheduler initialized (Port " +
str(scheduler_port) + ")...")
dask_client = Client("tcp://" + socket.gethostname() + ":" +
str(scheduler_port))
logger.write(" - Dask Client initialized...")
logger.write(" - Writing Dask Worker Job Files...")
with PyPostTools.cd(targetDir):
writeFile1 = PyPostTools.write_job_file(socket.gethostname(),
scheduler_port,
project_name="Nowcast",
queue="default",
nodes=dask_nodes,
wall_time=60,
nProcs=1)
writeFile2 = PyPostTools.write_worker_file(socket.gethostname(),
scheduler_port,
nProcs=1)
if (writeFile1 == False or writeFile2 == False):
dask_client.close()
logger.write(
" - Failed to write job files, are you missing an important parameter?"
)
sys.exit("")
return
else:
logger.write(
" - Dask Worker Job File Written, Submitting to Queue.")
PyPostTools.popen("chmod +x launch-worker.sh")
PyPostTools.popen("chmod +x dask-worker.job")
PyPostTools.popen("qsub dask-worker.job")
# Wait here for workers.
logger.write(" -> Worker Job submitted to queue, waiting for workers...")
while len(dask_client.scheduler_info()['workers']) < int(dask_nodes):
time.sleep(2)
logger.write(" -> Workers are now connected.")
#logger.write(" - Adding local packages to dask workers")
#dask_client.upload_file("PyPostTools.py")
#dask_client.upload_file("ArrayTools.py")
#dask_client.upload_file("Calculation.py")
#dask_client.upload_file("ColorMaps.py")
#dask_client.upload_file("Conversions.py")
#dask_client.upload_file("Plotting.py")
#dask_client.upload_file("PyPostSettings.py")
#dask_client.upload_file("Routines.py")
logger.write(" - Success!")
logger.write(" 1. Done.")
logger.write(" 2. Start Post-Processing Calculations")
calculation_future = start_calculations(dask_client, _routines,
dask_threads)
if (calculation_future != None):
wait(calculation_future)
result_calc = dask_client.gather(calculation_future)[0]
if (result_calc != 0):
logger.write(
"***FAIL*** An error occured in calculations method, check worker logs for more info."
)
logger.close()
sys.exit("")
logger.write(" 2. Done.")
logger.write(" 3. Generating Figures")
logger.write(" - Collecting files from target directory (" + targetDir +
").")
fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*"))
logger.write(" - " + str(len(fList3)) + " files have been found.")
logger.write(" -> Pushing run_plotting_routines() to dask.")
fullDict = _pySet.get_full_dict()
plotting_future = start_plotting(dask_client, fullDict, dask_threads)
wait(plotting_future)
result_plot = dask_client.gather(plotting_future)[0]
if (result_plot != 0):
logger.write(
"***FAIL*** An error occured in plotting method, check worker logs for more info."
)
logger.close()
sys.exit("")
logger.write(" 3. Done.")
logger.write(" 4. Final Steps")
logger.write(" 4. Done, Closing Dask Client.")
dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'],
close=True)
dask_client.close()
logger.write("All Steps Completed.")
logger.write("***SUCCESS*** Program execution complete.")
logger.close()