def publish_sdr(publisher, result_files):
# Now publish:
for result_file in result_files:
path, filename = os.path.split(result_file)
to_send = {}
# FIXME: Hardcoded machine name ! This is bad !!!
to_send['uri'] = ('ssh://safe.smhi.se/' + result_file)
to_send['filename'] = filename
to_send['instrument'] = 'viirs'
to_send['satellite'] = 'NPP'
to_send['format'] = 'HDF5'
to_send['type'] = 'SDR'
to_send['start_time'], to_send['end_time'] = get_sdr_times(filename)
msg = Message('/oper/polar/direct_readout/norrkoping',
"file", to_send).encode()
LOG.debug("sending: " + str(msg))
publisher.send(msg)
def publish_sdr(publisher, result_files):
# Now publish:
for result_file in result_files:
path, filename = os.path.split(result_file)
to_send = {}
# FIXME: Hardcoded machine name ! This is bad !!!
to_send['uri'] = ('ssh://safe.smhi.se/' + result_file)
to_send['filename'] = filename
to_send['instrument'] = 'viirs'
to_send['satellite'] = 'NPP'
to_send['format'] = 'HDF5'
to_send['type'] = 'SDR'
to_send['start_time'], to_send['end_time'] = get_sdr_times(filename)
msg = Message('/oper/polar/direct_readout/norrkoping', "file",
to_send).encode()
LOG.debug("sending: " + str(msg))
publisher.send(msg)
def npp_rolling_runner():
"""The NPP/VIIRS runner. Listens and triggers processing on RDR granules."""
level1_home = OPTIONS['level1_home']
working_dir = OPTIONS['working_dir']
with posttroll.subscriber.Subscribe('RDR') as subscr:
with Publish('npp_dr_runner', 'SDR', LEVEL1_PUBLISH_PORT) as publisher:
while True:
glist = []
pass_start_time = None
result_files = []
for msg in subscr.recv(timeout=90):
if msg is None and glist:
del glist[0]
keeper = glist[1]
LOG.info("Start CSPP: RDR files = " + str(glist))
run_cspp(*glist)
LOG.info("CSPP SDR processing finished...")
# Assume everything has gone well!
# Move the files from working dir:
new_result_files = get_files4pps(working_dir)
if len(new_result_files) == 0:
LOG.warning(
"No SDR files available. CSPP probably failed!"
)
continue
start_time = get_datetime(keeper)
start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
result_files.extend([
new_file for new_file in new_result_files
if start_str in new_file
])
publish_sdr(publisher, new_result_files)
break # end the loop and reinitialize !
LOG.debug("")
LOG.debug("\tMessage:")
LOG.debug(str(msg))
urlobj = urlparse(msg.data['uri'])
LOG.debug("Server = " + str(urlobj.netloc))
if urlobj.netloc != servername:
continue
LOG.info("Ok... " + str(urlobj.netloc))
LOG.info("Sat and Instrument: " +
str(msg.data['satellite']) + " " +
str(msg.data['instrument']))
if (msg.data['satellite'] == "NPP"
and msg.data['instrument'] == 'viirs'):
start_time = msg.data['start_time']
try:
orbnum = int(msg.data['orbit_number'])
except KeyError:
orbnum = None
rdr_filename = urlobj.path
path, fname = os.path.split(rdr_filename)
if fname.endswith('.h5'):
# Check if the file exists:
if not os.path.exists(rdr_filename):
raise IOError(
"File is reported to be dispatched " +
"but is not there! File = " + rdr_filename)
# Do processing:
LOG.info(
"RDR to SDR processing on npp/viirs with CSPP start!"
+ " Start time = ", start_time)
if orbnum:
LOG.info("Orb = %d" % orbnum)
LOG.info("File = %s" % str(rdr_filename))
LOG.info(
"Cleanup working dir before CSPP start...")
cleanup_cspp_workdir(working_dir)
# Fix orbit number in RDR file:
try:
rdr_filename = fix_rdrfile(rdr_filename)
except IOError:
LOG.error(
'Failed to fix orbit number in RDR file = '
+ str(urlobj.path))
import traceback
traceback.print_exc(file=sys.stderr)
glist.append(rdr_filename)
if len(glist) > 4:
raise RuntimeError(
"Invalid number of granules to "
"process!!!")
if len(glist) == 4:
del glist[0]
if len(glist) == 3:
keeper = glist[1]
if len(glist) == 2:
keeper = glist[0]
LOG.info("Start CSPP: RDR files = " + str(glist))
run_cspp(*glist)
LOG.info("CSPP SDR processing finished...")
# Assume everything has gone well!
# Move the files from working dir:
new_result_files = get_files4pps(working_dir)
if len(new_result_files) == 0:
LOG.warning(
"No SDR files available. CSPP probably failed!"
)
continue
result_files.extend(new_result_files)
start_time = get_datetime(keeper)
start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
result_files.extend([
new_file for new_file in new_result_files
if start_str in new_file
])
if pass_start_time is None:
pass_start_time = start_time
publish_sdr(publisher, new_result_files)
tobj = pass_start_time
LOG.info("Time used in sub-dir name: " +
str(tobj.strftime("%Y-%m-%d %H:%M")))
subd = create_pps_subdirname(tobj)
LOG.info("Create sub-directory for sdr files: %s" % str(subd))
pack_sdr_files4pps(result_files, subd)
make_okay_files(subd)
return
def npp_runner():
"""The NPP/VIIRS runner. Listens and triggers processing"""
level1_home = OPTIONS['level1_home']
working_dir = OPTIONS['working_dir']
with posttroll.subscriber.Subscribe('RDR') as subscr:
with Publish('npp_dr_runner', 'SDR', LEVEL1_PUBLISH_PORT) as publisher:
for msg in subscr.recv():
LOG.info("")
LOG.info("\tMessage:")
LOG.info(str(msg))
urlobj = urlparse(msg.data['uri'])
LOG.info("Server = " + str(urlobj.netloc))
if urlobj.netloc != servername:
continue
LOG.info("Ok... " + str(urlobj.netloc))
LOG.info("Sat and Instrument: " + str(msg.data['satellite']) +
" " + str(msg.data['instrument']))
if (msg.data['satellite'] == "NPP"
and msg.data['instrument'] == 'viirs'):
start_time = msg.data['start_time']
try:
orbnum = int(msg.data['orbit_number'])
except KeyError:
orbnum = None
path, fname = os.path.split(urlobj.path)
if fname.endswith('.h5'):
# Check if the file exists:
if not os.path.exists(urlobj.path):
raise IOError(
"File is reported to be dispatched " +
"but is not there! File = " + urlobj.path)
# Do processing:
LOG.info(
"RDR to SDR processing on npp/viirs with CSPP start!"
+ " Start time = ", start_time)
if orbnum:
LOG.info("Orb = %d" % orbnum)
LOG.info("File = %s" % str(urlobj.path))
LOG.info("Cleanup working dir before CSPP start...")
cleanup_cspp_workdir(working_dir)
# Fix orbit number in RDR file:
try:
rdr_filename = fix_rdrfile(urlobj.path)
except IOError:
LOG.error(
'Failed to fix orbit number in RDR file = ' +
str(urlobj.path))
import traceback
traceback.print_exc(file=sys.stderr)
LOG.info("Start CSPP: RDR file = " + str(rdr_filename))
run_cspp(rdr_filename)
LOG.info("CSPP SDR processing finished...")
# Assume everything has gone well!
# Move the files from working dir:
result_files = get_files4pps(working_dir)
if len(result_files) == 0:
LOG.warning(
"No SDR files available. CSPP probably failed!"
)
continue
#start_time = get_datetime(result_files[0])
# Use the start time from the RDR message!:
#tobj = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%S")
tobj = start_time
LOG.info("Time used in sub-dir name: " +
str(tobj.strftime("%Y-%m-%d %H:%M")))
subd = create_pps_subdirname(tobj)
LOG.info("Crate sub-directory for sdr files: %s" %
str(subd))
pack_sdr_files4pps(result_files, subd)
make_okay_files(subd)
# Now publish:
filename = result_files[0]
LOG.info("Filename = %s" % filename)
to_send = {}
# FIXME: Hardcoded machine name ! This is bad !!!
to_send['uri'] = ('ssh://safe.smhi.se/' +
os.path.join(level1_home, filename))
to_send['filename'] = filename
to_send['instrument'] = 'viirs'
to_send['satellite'] = 'NPP'
to_send['format'] = 'HDF5'
to_send['type'] = 'SDR'
to_send[
'start_time'] = start_time #start_time.isoformat()
msg = Message('/oper/polar/direct_readout/norrkoping',
"file", to_send).encode()
publisher.send(msg)
return
import os, sys
_NPP_PREPROC_LOG_FILE = os.environ.get('NPP_PREPROC_LOG_FILE', None)
import logging
if _NPP_PREPROC_LOG_FILE:
handler = logging.FileHandler(_NPP_PREPROC_LOG_FILE)
else:
handler = logging.StreamHandler(sys.stderr)
formatter = logging.Formatter(fmt=_DEFAULT_LOG_FORMAT,
datefmt=_DEFAULT_TIME_FORMAT)
handler.setFormatter(formatter)
handler.setLevel(10)
LOG.setLevel(10)
LOG.addHandler(handler)
CSPP_ENVS = {
"CSPP_HOME":
CSPP_HOME,
"CSPP_REV":
"20120215",
"CSPP_ANC_CACHE_DIR ":
os.path.join(CSPP_HOME, 'cache'),
"CSPP_ANC_HOME":
os.path.join(CSPP_HOME, 'static'),
"CSPP_ANC_TILE_PATH":
"%s/static/ADL/data/tiles/Terrain-Eco-ANC-Tile/withMetadata" % (CSPP_HOME),
"PATH":
'%s/atms/sdr:%s/atms/sdr:%s/viirs/edr:%s/viirs/sdr' %
def npp_rolling_runner():
"""The NPP/VIIRS runner. Listens and triggers processing on RDR granules."""
level1_home = OPTIONS['level1_home']
working_dir = OPTIONS['working_dir']
with posttroll.subscriber.Subscribe('RDR') as subscr:
with Publish('npp_dr_runner', 'SDR',
LEVEL1_PUBLISH_PORT) as publisher:
while True:
glist = []
pass_start_time = None
result_files = []
for msg in subscr.recv(timeout=90):
if msg is None and glist:
del glist[0]
keeper = glist[1]
LOG.info("Start CSPP: RDR files = " + str(glist))
run_cspp(*glist)
LOG.info("CSPP SDR processing finished...")
# Assume everything has gone well!
# Move the files from working dir:
new_result_files = get_files4pps(working_dir)
if len(new_result_files) == 0:
LOG.warning("No SDR files available. CSPP probably failed!")
continue
start_time = get_datetime(keeper)
start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
result_files.extend([new_file
for new_file in new_result_files
if start_str in new_file])
publish_sdr(publisher, new_result_files)
break # end the loop and reinitialize !
LOG.debug("")
LOG.debug("\tMessage:")
LOG.debug(str(msg))
urlobj = urlparse(msg.data['uri'])
LOG.debug("Server = " + str(urlobj.netloc))
if urlobj.netloc != servername:
continue
LOG.info("Ok... " + str(urlobj.netloc))
LOG.info("Sat and Instrument: " + str(msg.data['satellite'])
+ " " + str(msg.data['instrument']))
if (msg.data['satellite'] == "NPP" and
msg.data['instrument'] == 'viirs'):
start_time = msg.data['start_time']
try:
orbnum = int(msg.data['orbit_number'])
except KeyError:
orbnum = None
rdr_filename = urlobj.path
path, fname = os.path.split(rdr_filename)
if fname.endswith('.h5'):
# Check if the file exists:
if not os.path.exists(rdr_filename):
raise IOError("File is reported to be dispatched " +
"but is not there! File = " +
rdr_filename)
# Do processing:
LOG.info("RDR to SDR processing on npp/viirs with CSPP start!" +
" Start time = ", start_time)
if orbnum:
LOG.info("Orb = %d" % orbnum)
LOG.info("File = %s" % str(rdr_filename))
LOG.info("Cleanup working dir before CSPP start...")
cleanup_cspp_workdir(working_dir)
# Fix orbit number in RDR file:
try:
rdr_filename = fix_rdrfile(rdr_filename)
except IOError:
LOG.error('Failed to fix orbit number in RDR file = ' + str(urlobj.path))
import traceback
traceback.print_exc(file=sys.stderr)
glist.append(rdr_filename)
if len(glist) > 4:
raise RuntimeError("Invalid number of granules to "
"process!!!")
if len(glist) == 4:
del glist[0]
if len(glist) == 3:
keeper = glist[1]
if len(glist) == 2:
keeper = glist[0]
LOG.info("Start CSPP: RDR files = " + str(glist))
run_cspp(*glist)
LOG.info("CSPP SDR processing finished...")
# Assume everything has gone well!
# Move the files from working dir:
new_result_files = get_files4pps(working_dir)
if len(new_result_files) == 0:
LOG.warning("No SDR files available. CSPP probably failed!")
continue
result_files.extend(new_result_files)
start_time = get_datetime(keeper)
start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
result_files.extend([new_file
for new_file in new_result_files
if start_str in new_file])
if pass_start_time is None:
pass_start_time = start_time
publish_sdr(publisher, new_result_files)
tobj = pass_start_time
LOG.info("Time used in sub-dir name: " + str(tobj.strftime("%Y-%m-%d %H:%M")))
subd = create_pps_subdirname(tobj)
LOG.info("Create sub-directory for sdr files: %s" % str(subd))
pack_sdr_files4pps(result_files, subd)
make_okay_files(subd)
return
def npp_runner():
"""The NPP/VIIRS runner. Listens and triggers processing"""
level1_home = OPTIONS['level1_home']
working_dir = OPTIONS['working_dir']
with posttroll.subscriber.Subscribe('RDR') as subscr:
with Publish('npp_dr_runner', 'SDR',
LEVEL1_PUBLISH_PORT) as publisher:
for msg in subscr.recv():
LOG.info("")
LOG.info("\tMessage:")
LOG.info(str(msg))
urlobj = urlparse(msg.data['uri'])
LOG.info("Server = " + str(urlobj.netloc))
if urlobj.netloc != servername:
continue
LOG.info("Ok... " + str(urlobj.netloc))
LOG.info("Sat and Instrument: " + str(msg.data['satellite'])
+ " " + str(msg.data['instrument']))
if (msg.data['satellite'] == "NPP" and
msg.data['instrument'] == 'viirs'):
start_time = msg.data['start_time']
try:
orbnum = int(msg.data['orbit_number'])
except KeyError:
orbnum = None
path, fname = os.path.split(urlobj.path)
if fname.endswith('.h5'):
# Check if the file exists:
if not os.path.exists(urlobj.path):
raise IOError("File is reported to be dispatched " +
"but is not there! File = " +
urlobj.path)
# Do processing:
LOG.info("RDR to SDR processing on npp/viirs with CSPP start!" +
" Start time = ", start_time)
if orbnum:
LOG.info("Orb = %d" % orbnum)
LOG.info("File = %s" % str(urlobj.path))
LOG.info("Cleanup working dir before CSPP start...")
cleanup_cspp_workdir(working_dir)
# Fix orbit number in RDR file:
try:
rdr_filename = fix_rdrfile(urlobj.path)
except IOError:
LOG.error('Failed to fix orbit number in RDR file = ' + str(urlobj.path))
import traceback
traceback.print_exc(file=sys.stderr)
LOG.info("Start CSPP: RDR file = " + str(rdr_filename))
run_cspp(rdr_filename)
LOG.info("CSPP SDR processing finished...")
# Assume everything has gone well!
# Move the files from working dir:
result_files = get_files4pps(working_dir)
if len(result_files) == 0:
LOG.warning("No SDR files available. CSPP probably failed!")
continue
#start_time = get_datetime(result_files[0])
# Use the start time from the RDR message!:
#tobj = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%S")
tobj = start_time
LOG.info("Time used in sub-dir name: " + str(tobj.strftime("%Y-%m-%d %H:%M")))
subd = create_pps_subdirname(tobj)
LOG.info("Crate sub-directory for sdr files: %s" % str(subd))
pack_sdr_files4pps(result_files, subd)
make_okay_files(subd)
# Now publish:
filename = result_files[0]
LOG.info("Filename = %s" % filename)
to_send = {}
# FIXME: Hardcoded machine name ! This is bad !!!
to_send['uri'] = ('ssh://safe.smhi.se/' +
os.path.join(level1_home,
filename))
to_send['filename'] = filename
to_send['instrument'] = 'viirs'
to_send['satellite'] = 'NPP'
to_send['format'] = 'HDF5'
to_send['type'] = 'SDR'
to_send['start_time'] = start_time #start_time.isoformat()
msg = Message('/oper/polar/direct_readout/norrkoping',
"file", to_send).encode()
publisher.send(msg)
return
import os, sys
_NPP_PREPROC_LOG_FILE = os.environ.get('NPP_PREPROC_LOG_FILE', None)
import logging
if _NPP_PREPROC_LOG_FILE:
handler = logging.FileHandler(_NPP_PREPROC_LOG_FILE)
else:
handler = logging.StreamHandler(sys.stderr)
formatter = logging.Formatter(fmt=_DEFAULT_LOG_FORMAT,
datefmt=_DEFAULT_TIME_FORMAT)
handler.setFormatter(formatter)
handler.setLevel(10)
LOG.setLevel(10)
LOG.addHandler(handler)
CSPP_ENVS = {"CSPP_HOME": CSPP_HOME,
"CSPP_REV": "20120215",
"CSPP_ANC_CACHE_DIR " : os.path.join(CSPP_HOME,'cache'),
"CSPP_ANC_HOME" : os.path.join(CSPP_HOME,'static'),
"CSPP_ANC_TILE_PATH" : "%s/static/ADL/data/tiles/Terrain-Eco-ANC-Tile/withMetadata" % (CSPP_HOME),
"PATH" : '%s/atms/sdr:%s/atms/sdr:%s/viirs/edr:%s/viirs/sdr' % (CSPP_HOME,
CSPP_HOME,
CSPP_HOME,
CSPP_HOME),
"ADL_HOME" : "%s/ADL" % (CSPP_HOME),
"NPP_GRANULE_ID_BASETIME" : "1698019234000000",