def prepare_nwp4pps(flens):
"""Prepare NWP data for pps"""
starttime = datetime.utcnow() - timedelta(days=1)
try:
update_nwp(starttime, flens)
LOG.info("Ready with nwp preparation")
LOG.debug("Leaving prepare_nwp4pps...")
except Exception:
LOG.exception("Something went wrong in update_nwp...")
raise
def prepare_nwp4pps(sphor_obj, starttime, flens):
"""Prepare NWP data for pps"""
try:
LOG.debug("Waiting for acquired semaphore for nwp prepare...")
with sphor_obj:
LOG.debug("Acquired semaphore for nwp preparation...")
update_nwp(starttime, flens)
LOG.info("Ready with nwp preparation")
LOG.debug("Leaving prepare_nwp4pps...")
except:
LOG.exception("Something went wrong in update_nwp...")
raise
def prepare_nwp4pps(flens, nwp_handeling_module):
"""Prepare NWP data for pps."""
starttime = datetime.utcnow() - timedelta(days=1)
if nwp_handeling_module:
LOG.debug(
"Use custom nwp_handeling_function provided in config file...")
LOG.debug("nwp_module_name = %s", str(nwp_handeling_module))
try:
name = "update_nwp"
name = name.replace("/", "")
module = __import__(nwp_handeling_module, globals(), locals(),
[name])
LOG.info("function : {} loaded from module: {}".format(
[name], nwp_handeling_module))
except (ImportError, ModuleNotFoundError):
LOG.exception("Failed to import custom compositer for %s",
str(name))
raise
try:
params = {}
params['starttime'] = starttime
params['nlengths'] = flens
params['options'] = OPTIONS
getattr(module, name)(params)
except AttributeError:
LOG.debug("Could not get attribute %s from %s", str(name),
str(module))
else:
LOG.debug(
"No custom nwp_handeling_function provided in config file...")
LOG.debug("Use build in.")
try:
update_nwp(starttime, flens)
except (NwpPrepareError, IOError):
LOG.exception("Something went wrong in update_nwp...")
raise
LOG.info("Ready with nwp preparation")
LOG.debug("Leaving prepare_nwp4pps...")
def pps(options):
"""The PPS runner. Triggers processing of PPS main script once AAPP or CSPP
is ready with a level-1 file"""
LOG.info("*** Start the PPS level-2 runner:")
LOG.info("First check if NWP data should be downloaded and prepared")
now = datetime.utcnow()
update_nwp(now - timedelta(days=1), NWP_FLENS)
LOG.info("Ready with nwp preparation...")
listener_q = Queue.Queue()
publisher_q = Queue.Queue()
pub_thread = FilePublisher(publisher_q,
options['publish_topic'],
runner_name='pps_runner')
pub_thread.start()
listen_thread = FileListener(listener_q, options['subscribe_topics'])
listen_thread.start()
files4pps = {}
thread_pool = ThreadPool(options['number_of_threads'])
while True:
try:
msg = listener_q.get()
except Queue.Empty:
continue
LOG.debug("Number of threads currently alive: " +
str(threading.active_count()))
orbit_number = int(msg.data['orbit_number'])
platform_name = msg.data['platform_name']
starttime = msg.data['start_time']
endtime = msg.data['end_time']
satday = starttime.strftime('%Y%m%d')
sathour = starttime.strftime('%H%M')
sensors = SENSOR_LIST.get(platform_name, None)
scene = {
'platform_name': platform_name,
'orbit_number': orbit_number,
'satday': satday,
'sathour': sathour,
'starttime': starttime,
'endtime': endtime,
'sensor': sensors
}
status = ready2run(msg, files4pps)
if status:
LOG.info('Start a thread preparing the nwp data and run pps...')
thread_pool.new_thread(message_uid(msg),
target=run_nwp_and_pps,
args=(scene, NWP_FLENS, publisher_q, msg,
options))
LOG.debug("Number of threads currently alive: " +
str(threading.active_count()))
pub_thread.stop()
listen_thread.stop()
def pps():
"""The PPS runner. Triggers processing of PPS main script once AAPP or CSPP
is ready with a level-1 file"""
LOG.info("*** Start the PPS level-2 runner:")
LOG.info("First check if NWP data should be downloaded and prepared")
now = datetime.utcnow()
update_nwp(now - timedelta(days=1), NWP_FLENS)
LOG.info("Ready with nwp preparation...")
nwp_pp_sema = threading.Semaphore(1)
sema = threading.Semaphore(5)
listener_q = Queue.Queue()
publisher_q = Queue.Queue()
pub_thread = FilePublisher(publisher_q)
pub_thread.start()
listen_thread = FileListener(listener_q)
listen_thread.start()
files4pps = {}
threads = []
jobs_dict = {}
while True:
try:
msg = listener_q.get()
except Queue.Empty:
continue
LOG.debug("Number of threads currently alive: " +
str(threading.active_count()))
orbit_number = int(msg.data['orbit_number'])
platform_name = msg.data['platform_name']
starttime = msg.data['start_time']
endtime = msg.data['end_time']
keyname = (str(platform_name) + '_' + str(orbit_number) + '_' +
str(starttime.strftime('%Y%m%d%H%M')))
status = ready2run(msg, files4pps, jobs_dict, keyname)
if status:
# Process pps on the scene
sensors = SENSOR_LIST.get(platform_name, None)
satday = starttime.strftime('%Y%m%d')
sathour = starttime.strftime('%H%M')
scene = {
'platform_name': platform_name,
'orbit_number': orbit_number,
'satday': satday,
'sathour': sathour,
'starttime': starttime,
'endtime': endtime,
'sensor': sensors
}
if keyname not in jobs_dict:
LOG.warning("Scene-run seems unregistered! Forget it...")
continue
LOG.info('Start a thread preparing the nwp data...')
now = datetime.utcnow()
t_nwp_pp = threading.Thread(target=prepare_nwp4pps,
args=(nwp_pp_sema,
now - timedelta(days=1),
NWP_FLENS))
t_nwp_pp.start()
t__ = threading.Thread(target=pps_worker,
args=(sema, scene, jobs_dict[keyname],
publisher_q, msg))
threads.append(t__)
t__.start()
# Clean the files4pps dict:
LOG.debug("files4pps: " + str(files4pps))
try:
files4pps.pop(keyname)
except KeyError:
LOG.warning("Failed trying to remove key " + str(keyname) +
" from dictionary files4pps")
LOG.debug("After cleaning: files4pps = " + str(files4pps))
LOG.debug("Number of threads currently alive: " +
str(threading.active_count()))
# Block any future run on this scene for x minutes from now
# x = 20
thread_job_registry = threading.Timer(20 * 60.0,
reset_job_registry,
args=(jobs_dict, keyname))
thread_job_registry.start()
LOG.info("Wait till all threads are dead...")
while True:
workers_ready = True
for thread in threads:
if thread.is_alive():
workers_ready = False
if workers_ready:
break
pub_thread.stop()
listen_thread.stop()
return