Пример #1
0
def launch_python_post():
	curDir = os.path.dirname(os.path.abspath(__file__)) 
	logger = PyPostTools.pyPostLogger()

	logger.write("Initializing WRF Python Post-Processing Program")
	#Step 1: Load program settings
	logger.write(" 1. Application Initalization")
	logger.write("  - Loading control file, python_post_control.txt")
	_pySet = PyPostSettings.PyPostSettings()
	logger.write("  - Success!")
	logger.write("  - Testing Environmental Variables")
	try:
		dask_nodes = os.environ["PYTHON_POST_NODES"]
		dask_threads = os.environ["PYTHON_POST_THREADS"]	
		postDir = os.environ["PYTHON_POST_DIR"]
		targetDir = os.environ["PYTHON_POST_TARG_DIR"]
	except KeyError:
		logger.write("***FAIL*** KeyError encountered while trying to access important environmental variables, abort.")
		sys.exit("")
	logger.write("  - Success!")
	logger.write("  - Initializing Dask (" + str(dask_nodes) + " Nodes Requested), Collecting routines needed")
	_routines = Routines.Routines()
	logger.write("   - Async IO Loop initialized...")	
	def f(scheduler_port):
		async def g(port):
			s = Scheduler(port=scheduler_port)
			await s
			await s.finished()
		asyncio.get_event_loop().run_until_complete(g(scheduler_port))
	# Starts the scheduler in its own process - needed as otherwise it will 
	# occupy the program and make it do an infinite loop
	process = Process(target=f, args=(scheduler_port,))
	process.start()
	logger.write("   - Dask Scheduler initialized (Port " + str(scheduler_port) + ")...")
	try:
		dask_client = Client("tcp://" + socket.gethostname() + ":" + str(scheduler_port), timeout=30)
	except OSError:
		logger.write("  <-> Dask Client could not be created, timeout error.")
		process.terminate()
		sys.exit()
	logger.write("   - Dask Client initialized...")
	logger.write("   - Writing Dask Worker Job Files...")
	with PyPostTools.cd(targetDir):
		writeFile = PyPostTools.write_job_file(socket.gethostname(), scheduler_port, project_name="climate_severe", queue="debug-cache-quad", nodes=dask_nodes, wall_time=60, nProcs=1)
		if(writeFile == False):
			dask_client.close()
			logger.write("   - Failed to write job file, are you missing an important parameter?")
			sys.exit("")
			return
		else:
			logger.write("   - Dask Worker Job File Written, Submitting to Queue.")
			PyPostTools.popen("chmod +x dask-worker.job")
			PyPostTools.popen("qsub dask-worker.job")
	# Wait here for workers.
	logger.write("   -> Worker Job submitted to queue, waiting for workers...")
	while len(dask_client.scheduler_info()['workers']) < int(dask_nodes):
		time.sleep(2)
	logger.write("   -> Workers are now connected.")
	logger.write("  - Success!")
	logger.write(" 1. Done.")
	logger.write(" 2. Start Post-Processing Calculations")
	start_calculations(dask_client, _routines, dask_threads, process)
	logger.write(" 2. Done.")
	logger.write(" 3. Generating Figures")
	logger.write("  - Collecting files from target directory (" + targetDir + ").")
	fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*"))
	logger.write("  - " + str(len(fList3)) + " files have been found.")
	logger.write(" -> Pushing run_plotting_routines() to dask.")
	fullDict = _pySet.get_full_dict()
	plotting_future = start_plotting(dask_client, fullDict, dask_threads, process)
	wait(plotting_future)
	result_plot = dask_client.gather(plotting_future)[0]
	if(result_plot != 0):
		logger.write("***FAIL*** An error occured in plotting method, check worker logs for more info.")
		logger.close()
		sys.exit("")	
	logger.write(" 3. Done.")
	logger.write(" 4. Final Steps")
	
	logger.write(" 4. Done, Closing Dask Client.")
	# Close the client object
	dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'], close=True)
	dask_client.close()	
	logger.write("All Steps Completed.")
	logger.write("***SUCCESS*** Program execution complete.")
	logger.close()
	del dask_client
	process.terminate()
Пример #2
0
def launch_python_post():
    curDir = os.path.dirname(os.path.abspath(__file__))
    logger = PyPostTools.pyPostLogger()

    logger.write("Initializing WRF Python Post-Processing Program")
    #Step 1: Load program settings
    logger.write(" 1. Application Initalization")
    logger.write("  - Loading control file, python_post_control.txt")
    _pySet = PyPostSettings.PyPostSettings()
    logger.write("  - Success!")
    logger.write("  - Testing Environmental Variables")
    try:
        dask_nodes = os.environ["PYTHON_POST_NODES"]
        dask_threads = os.environ["PYTHON_POST_THREADS"]
        postDir = os.environ["PYTHON_POST_DIR"]
        targetDir = os.environ["PYTHON_POST_TARG_DIR"]
    except KeyError:
        logger.write(
            "***FAIL*** KeyError encountered while trying to access important environmental variables, abort."
        )
        sys.exit("")
    logger.write("  - Success!")
    logger.write("  - Initializing Dask (" + str(dask_nodes) +
                 " Nodes Requested), Collecting routines needed")
    _routines = Routines.Routines()
    # Start Dask Tasks
    #cLoop = IOLoop.current()
    #t = Thread(target = cLoop.start, daemon = True)
    #t.start()

    logger.write("   - Async IO Loop initialized...")

    async def f(port):
        s = Scheduler(port=scheduler_port)
        s = await s
        await s.finished()
        return 1

    asyncio.gather(f(scheduler_port))

    #asyncio.get_event_loop().run_until_complete(f(scheduler_port))

    logger.write("   - Dask Scheduler initialized (Port " +
                 str(scheduler_port) + ")...")
    dask_client = Client("tcp://" + socket.gethostname() + ":" +
                         str(scheduler_port))
    logger.write("   - Dask Client initialized...")
    logger.write("   - Writing Dask Worker Job Files...")
    with PyPostTools.cd(targetDir):
        writeFile1 = PyPostTools.write_job_file(socket.gethostname(),
                                                scheduler_port,
                                                project_name="Nowcast",
                                                queue="default",
                                                nodes=dask_nodes,
                                                wall_time=60,
                                                nProcs=1)
        writeFile2 = PyPostTools.write_worker_file(socket.gethostname(),
                                                   scheduler_port,
                                                   nProcs=1)
        if (writeFile1 == False or writeFile2 == False):
            dask_client.close()
            logger.write(
                "   - Failed to write job files, are you missing an important parameter?"
            )
            sys.exit("")
            return
        else:
            logger.write(
                "   - Dask Worker Job File Written, Submitting to Queue.")
            PyPostTools.popen("chmod +x launch-worker.sh")
            PyPostTools.popen("chmod +x dask-worker.job")
            PyPostTools.popen("qsub dask-worker.job")
    # Wait here for workers.
    logger.write("   -> Worker Job submitted to queue, waiting for workers...")
    while len(dask_client.scheduler_info()['workers']) < int(dask_nodes):
        time.sleep(2)
    logger.write("   -> Workers are now connected.")
    #logger.write("   - Adding local packages to dask workers")
    #dask_client.upload_file("PyPostTools.py")
    #dask_client.upload_file("ArrayTools.py")
    #dask_client.upload_file("Calculation.py")
    #dask_client.upload_file("ColorMaps.py")
    #dask_client.upload_file("Conversions.py")
    #dask_client.upload_file("Plotting.py")
    #dask_client.upload_file("PyPostSettings.py")
    #dask_client.upload_file("Routines.py")
    logger.write("  - Success!")
    logger.write(" 1. Done.")
    logger.write(" 2. Start Post-Processing Calculations")
    calculation_future = start_calculations(dask_client, _routines,
                                            dask_threads)
    if (calculation_future != None):
        wait(calculation_future)
        result_calc = dask_client.gather(calculation_future)[0]
        if (result_calc != 0):
            logger.write(
                "***FAIL*** An error occured in calculations method, check worker logs for more info."
            )
            logger.close()
            sys.exit("")
    logger.write(" 2. Done.")
    logger.write(" 3. Generating Figures")
    logger.write("  - Collecting files from target directory (" + targetDir +
                 ").")
    fList3 = sorted(glob.glob(targetDir + "WRFPRS_F*"))
    logger.write("  - " + str(len(fList3)) + " files have been found.")
    logger.write(" -> Pushing run_plotting_routines() to dask.")
    fullDict = _pySet.get_full_dict()
    plotting_future = start_plotting(dask_client, fullDict, dask_threads)
    wait(plotting_future)
    result_plot = dask_client.gather(plotting_future)[0]
    if (result_plot != 0):
        logger.write(
            "***FAIL*** An error occured in plotting method, check worker logs for more info."
        )
        logger.close()
        sys.exit("")
    logger.write(" 3. Done.")
    logger.write(" 4. Final Steps")

    logger.write(" 4. Done, Closing Dask Client.")
    dask_client.retire_workers(workers=dask_client.scheduler_info()['workers'],
                               close=True)
    dask_client.close()
    logger.write("All Steps Completed.")
    logger.write("***SUCCESS*** Program execution complete.")
    logger.close()