def publish_sdr(publisher, result_files):
    # Now publish:
    for result_file in result_files:
        path, filename = os.path.split(result_file)
        to_send = {}
        # FIXME: Hardcoded machine name ! This is bad !!!
        to_send['uri'] = ('ssh://safe.smhi.se/' + result_file)
        to_send['filename'] = filename
        to_send['instrument'] = 'viirs'
        to_send['satellite'] = 'NPP'
        to_send['format'] = 'HDF5'
        to_send['type'] = 'SDR'
        
        to_send['start_time'], to_send['end_time'] = get_sdr_times(filename)
        msg = Message('/oper/polar/direct_readout/norrkoping',
                      "file", to_send).encode()
        LOG.debug("sending: " + str(msg))
        publisher.send(msg)
Exemple #2
0
def publish_sdr(publisher, result_files):
    # Now publish:
    for result_file in result_files:
        path, filename = os.path.split(result_file)
        to_send = {}
        # FIXME: Hardcoded machine name ! This is bad !!!
        to_send['uri'] = ('ssh://safe.smhi.se/' + result_file)
        to_send['filename'] = filename
        to_send['instrument'] = 'viirs'
        to_send['satellite'] = 'NPP'
        to_send['format'] = 'HDF5'
        to_send['type'] = 'SDR'

        to_send['start_time'], to_send['end_time'] = get_sdr_times(filename)
        msg = Message('/oper/polar/direct_readout/norrkoping', "file",
                      to_send).encode()
        LOG.debug("sending: " + str(msg))
        publisher.send(msg)
Exemple #3
0
def npp_rolling_runner():
    """The NPP/VIIRS runner. Listens and triggers processing on RDR granules."""

    level1_home = OPTIONS['level1_home']
    working_dir = OPTIONS['working_dir']

    with posttroll.subscriber.Subscribe('RDR') as subscr:
        with Publish('npp_dr_runner', 'SDR', LEVEL1_PUBLISH_PORT) as publisher:
            while True:
                glist = []
                pass_start_time = None
                result_files = []
                for msg in subscr.recv(timeout=90):
                    if msg is None and glist:
                        del glist[0]
                        keeper = glist[1]
                        LOG.info("Start CSPP: RDR files = " + str(glist))
                        run_cspp(*glist)
                        LOG.info("CSPP SDR processing finished...")
                        # Assume everything has gone well!
                        # Move the files from working dir:
                        new_result_files = get_files4pps(working_dir)
                        if len(new_result_files) == 0:
                            LOG.warning(
                                "No SDR files available. CSPP probably failed!"
                            )
                            continue

                        start_time = get_datetime(keeper)
                        start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
                        result_files.extend([
                            new_file for new_file in new_result_files
                            if start_str in new_file
                        ])
                        publish_sdr(publisher, new_result_files)
                        break  # end the loop and reinitialize !

                    LOG.debug("")
                    LOG.debug("\tMessage:")
                    LOG.debug(str(msg))
                    urlobj = urlparse(msg.data['uri'])
                    LOG.debug("Server = " + str(urlobj.netloc))
                    if urlobj.netloc != servername:
                        continue
                    LOG.info("Ok... " + str(urlobj.netloc))
                    LOG.info("Sat and Instrument: " +
                             str(msg.data['satellite']) + " " +
                             str(msg.data['instrument']))

                    if (msg.data['satellite'] == "NPP"
                            and msg.data['instrument'] == 'viirs'):
                        start_time = msg.data['start_time']
                        try:
                            orbnum = int(msg.data['orbit_number'])
                        except KeyError:
                            orbnum = None
                        rdr_filename = urlobj.path
                        path, fname = os.path.split(rdr_filename)
                        if fname.endswith('.h5'):
                            # Check if the file exists:
                            if not os.path.exists(rdr_filename):
                                raise IOError(
                                    "File is reported to be dispatched " +
                                    "but is not there! File = " + rdr_filename)

                            # Do processing:
                            LOG.info(
                                "RDR to SDR processing on npp/viirs with CSPP start!"
                                + " Start time = ", start_time)
                            if orbnum:
                                LOG.info("Orb = %d" % orbnum)
                            LOG.info("File = %s" % str(rdr_filename))
                            LOG.info(
                                "Cleanup working dir before CSPP start...")
                            cleanup_cspp_workdir(working_dir)

                            # Fix orbit number in RDR file:
                            try:
                                rdr_filename = fix_rdrfile(rdr_filename)
                            except IOError:
                                LOG.error(
                                    'Failed to fix orbit number in RDR file = '
                                    + str(urlobj.path))
                                import traceback
                                traceback.print_exc(file=sys.stderr)

                            glist.append(rdr_filename)

                            if len(glist) > 4:
                                raise RuntimeError(
                                    "Invalid number of granules to "
                                    "process!!!")
                            if len(glist) == 4:
                                del glist[0]
                            if len(glist) == 3:
                                keeper = glist[1]
                            if len(glist) == 2:
                                keeper = glist[0]

                            LOG.info("Start CSPP: RDR files = " + str(glist))
                            run_cspp(*glist)
                            LOG.info("CSPP SDR processing finished...")
                            # Assume everything has gone well!
                            # Move the files from working dir:
                            new_result_files = get_files4pps(working_dir)
                            if len(new_result_files) == 0:
                                LOG.warning(
                                    "No SDR files available. CSPP probably failed!"
                                )
                                continue

                            result_files.extend(new_result_files)

                            start_time = get_datetime(keeper)
                            start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
                            result_files.extend([
                                new_file for new_file in new_result_files
                                if start_str in new_file
                            ])
                            if pass_start_time is None:
                                pass_start_time = start_time

                            publish_sdr(publisher, new_result_files)

                tobj = pass_start_time
                LOG.info("Time used in sub-dir name: " +
                         str(tobj.strftime("%Y-%m-%d %H:%M")))
                subd = create_pps_subdirname(tobj)
                LOG.info("Create sub-directory for sdr files: %s" % str(subd))
                pack_sdr_files4pps(result_files, subd)
                make_okay_files(subd)

    return
Exemple #4
0
def npp_runner():
    """The NPP/VIIRS runner. Listens and triggers processing"""

    level1_home = OPTIONS['level1_home']
    working_dir = OPTIONS['working_dir']

    with posttroll.subscriber.Subscribe('RDR') as subscr:
        with Publish('npp_dr_runner', 'SDR', LEVEL1_PUBLISH_PORT) as publisher:
            for msg in subscr.recv():
                LOG.info("")
                LOG.info("\tMessage:")
                LOG.info(str(msg))
                urlobj = urlparse(msg.data['uri'])
                LOG.info("Server = " + str(urlobj.netloc))
                if urlobj.netloc != servername:
                    continue
                LOG.info("Ok... " + str(urlobj.netloc))
                LOG.info("Sat and Instrument: " + str(msg.data['satellite']) +
                         " " + str(msg.data['instrument']))

                if (msg.data['satellite'] == "NPP"
                        and msg.data['instrument'] == 'viirs'):
                    start_time = msg.data['start_time']
                    try:
                        orbnum = int(msg.data['orbit_number'])
                    except KeyError:
                        orbnum = None
                    path, fname = os.path.split(urlobj.path)
                    if fname.endswith('.h5'):
                        # Check if the file exists:
                        if not os.path.exists(urlobj.path):
                            raise IOError(
                                "File is reported to be dispatched " +
                                "but is not there! File = " + urlobj.path)

                        # Do processing:
                        LOG.info(
                            "RDR to SDR processing on npp/viirs with CSPP start!"
                            + " Start time = ", start_time)
                        if orbnum:
                            LOG.info("Orb = %d" % orbnum)
                        LOG.info("File = %s" % str(urlobj.path))
                        LOG.info("Cleanup working dir before CSPP start...")
                        cleanup_cspp_workdir(working_dir)
                        # Fix orbit number in RDR file:
                        try:
                            rdr_filename = fix_rdrfile(urlobj.path)
                        except IOError:
                            LOG.error(
                                'Failed to fix orbit number in RDR file = ' +
                                str(urlobj.path))
                            import traceback
                            traceback.print_exc(file=sys.stderr)

                        LOG.info("Start CSPP: RDR file = " + str(rdr_filename))
                        run_cspp(rdr_filename)
                        LOG.info("CSPP SDR processing finished...")
                        # Assume everything has gone well!
                        # Move the files from working dir:
                        result_files = get_files4pps(working_dir)
                        if len(result_files) == 0:
                            LOG.warning(
                                "No SDR files available. CSPP probably failed!"
                            )
                            continue

                        #start_time = get_datetime(result_files[0])
                        # Use the start time from the RDR message!:
                        #tobj = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%S")
                        tobj = start_time
                        LOG.info("Time used in sub-dir name: " +
                                 str(tobj.strftime("%Y-%m-%d %H:%M")))
                        subd = create_pps_subdirname(tobj)
                        LOG.info("Crate sub-directory for sdr files: %s" %
                                 str(subd))
                        pack_sdr_files4pps(result_files, subd)
                        make_okay_files(subd)

                        # Now publish:
                        filename = result_files[0]
                        LOG.info("Filename = %s" % filename)
                        to_send = {}
                        # FIXME: Hardcoded machine name ! This is bad !!!
                        to_send['uri'] = ('ssh://safe.smhi.se/' +
                                          os.path.join(level1_home, filename))
                        to_send['filename'] = filename
                        to_send['instrument'] = 'viirs'
                        to_send['satellite'] = 'NPP'
                        to_send['format'] = 'HDF5'
                        to_send['type'] = 'SDR'
                        to_send[
                            'start_time'] = start_time  #start_time.isoformat()
                        msg = Message('/oper/polar/direct_readout/norrkoping',
                                      "file", to_send).encode()
                        publisher.send(msg)

    return
Exemple #5
0
import os, sys
_NPP_PREPROC_LOG_FILE = os.environ.get('NPP_PREPROC_LOG_FILE', None)
import logging

if _NPP_PREPROC_LOG_FILE:
    handler = logging.FileHandler(_NPP_PREPROC_LOG_FILE)
else:
    handler = logging.StreamHandler(sys.stderr)

formatter = logging.Formatter(fmt=_DEFAULT_LOG_FORMAT,
                              datefmt=_DEFAULT_TIME_FORMAT)
handler.setFormatter(formatter)

handler.setLevel(10)
LOG.setLevel(10)
LOG.addHandler(handler)

CSPP_ENVS = {
    "CSPP_HOME":
    CSPP_HOME,
    "CSPP_REV":
    "20120215",
    "CSPP_ANC_CACHE_DIR ":
    os.path.join(CSPP_HOME, 'cache'),
    "CSPP_ANC_HOME":
    os.path.join(CSPP_HOME, 'static'),
    "CSPP_ANC_TILE_PATH":
    "%s/static/ADL/data/tiles/Terrain-Eco-ANC-Tile/withMetadata" % (CSPP_HOME),
    "PATH":
    '%s/atms/sdr:%s/atms/sdr:%s/viirs/edr:%s/viirs/sdr' %
def npp_rolling_runner():
    """The NPP/VIIRS runner. Listens and triggers processing on RDR granules."""

    level1_home = OPTIONS['level1_home']
    working_dir = OPTIONS['working_dir']

    with posttroll.subscriber.Subscribe('RDR') as subscr:
        with Publish('npp_dr_runner', 'SDR', 
                     LEVEL1_PUBLISH_PORT) as publisher:
            while True:
                glist = []
                pass_start_time = None
                result_files = []
                for msg in subscr.recv(timeout=90):
                    if msg is None and glist:
                        del glist[0]
                        keeper = glist[1]
                        LOG.info("Start CSPP: RDR files = " + str(glist))
                        run_cspp(*glist)
                        LOG.info("CSPP SDR processing finished...")
                        # Assume everything has gone well! 
                        # Move the files from working dir:
                        new_result_files = get_files4pps(working_dir)
                        if len(new_result_files) == 0:
                            LOG.warning("No SDR files available. CSPP probably failed!")
                            continue


                        start_time = get_datetime(keeper)
                        start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
                        result_files.extend([new_file
                                             for new_file in new_result_files
                                             if start_str in new_file])
                        publish_sdr(publisher, new_result_files)
                        break # end the loop and reinitialize !

                    LOG.debug("")
                    LOG.debug("\tMessage:")
                    LOG.debug(str(msg))
                    urlobj = urlparse(msg.data['uri'])
                    LOG.debug("Server = " + str(urlobj.netloc))
                    if urlobj.netloc != servername:
                        continue
                    LOG.info("Ok... " + str(urlobj.netloc))
                    LOG.info("Sat and Instrument: " + str(msg.data['satellite']) 
                             + " " + str(msg.data['instrument']))

                    if (msg.data['satellite'] == "NPP" and 
                        msg.data['instrument'] == 'viirs'):
                        start_time = msg.data['start_time']
                        try:
                            orbnum = int(msg.data['orbit_number'])            
                        except KeyError:
                            orbnum = None
                        rdr_filename = urlobj.path
                        path, fname =  os.path.split(rdr_filename)
                        if fname.endswith('.h5'):
                            # Check if the file exists:
                            if not os.path.exists(rdr_filename):
                                raise IOError("File is reported to be dispatched " + 
                                              "but is not there! File = " + 
                                              rdr_filename)

                            # Do processing:
                            LOG.info("RDR to SDR processing on npp/viirs with CSPP start!" + 
                                     " Start time = ", start_time)
                            if orbnum:
                                LOG.info("Orb = %d" % orbnum)
                            LOG.info("File = %s" % str(rdr_filename))
                            LOG.info("Cleanup working dir before CSPP start...")
                            cleanup_cspp_workdir(working_dir)

                            # Fix orbit number in RDR file:
                            try:
                                rdr_filename = fix_rdrfile(rdr_filename)
                            except IOError:
                                LOG.error('Failed to fix orbit number in RDR file = ' + str(urlobj.path))
                                import traceback
                                traceback.print_exc(file=sys.stderr)

                            glist.append(rdr_filename)

                            if len(glist) > 4:
                                raise RuntimeError("Invalid number of granules to "
                                                   "process!!!")
                            if len(glist) == 4:
                                del glist[0]
                            if len(glist) == 3:
                                keeper = glist[1]
                            if len(glist) == 2:
                                keeper = glist[0]

                            LOG.info("Start CSPP: RDR files = " + str(glist))
                            run_cspp(*glist)
                            LOG.info("CSPP SDR processing finished...")
                            # Assume everything has gone well! 
                            # Move the files from working dir:
                            new_result_files = get_files4pps(working_dir)
                            if len(new_result_files) == 0:
                                LOG.warning("No SDR files available. CSPP probably failed!")
                                continue

                            result_files.extend(new_result_files)
                            
                            start_time = get_datetime(keeper)
                            start_str = start_time.strftime("d%Y%m%d_t%H%M%S")
                            result_files.extend([new_file
                                                 for new_file in new_result_files
                                                 if start_str in new_file])
                            if pass_start_time is None:
                                pass_start_time = start_time

                            publish_sdr(publisher, new_result_files)
                            
                        
                tobj = pass_start_time
                LOG.info("Time used in sub-dir name: " + str(tobj.strftime("%Y-%m-%d %H:%M")))
                subd = create_pps_subdirname(tobj)
                LOG.info("Create sub-directory for sdr files: %s" % str(subd))
                pack_sdr_files4pps(result_files, subd)
                make_okay_files(subd)



    return
def npp_runner():
    """The NPP/VIIRS runner. Listens and triggers processing"""

    level1_home = OPTIONS['level1_home']
    working_dir = OPTIONS['working_dir']

    with posttroll.subscriber.Subscribe('RDR') as subscr:
        with Publish('npp_dr_runner', 'SDR', 
                     LEVEL1_PUBLISH_PORT) as publisher:        
            for msg in subscr.recv():
                LOG.info("")
                LOG.info("\tMessage:")
                LOG.info(str(msg))
                urlobj = urlparse(msg.data['uri'])
                LOG.info("Server = " + str(urlobj.netloc))
                if urlobj.netloc != servername:
                    continue
                LOG.info("Ok... " + str(urlobj.netloc))
                LOG.info("Sat and Instrument: " + str(msg.data['satellite']) 
                         + " " + str(msg.data['instrument']))

                if (msg.data['satellite'] == "NPP" and 
                    msg.data['instrument'] == 'viirs'):
                    start_time = msg.data['start_time']
                    try:
                        orbnum = int(msg.data['orbit_number'])            
                    except KeyError:
                        orbnum = None
                    path, fname =  os.path.split(urlobj.path)
                    if fname.endswith('.h5'):
                        # Check if the file exists:
                        if not os.path.exists(urlobj.path):
                            raise IOError("File is reported to be dispatched " + 
                                          "but is not there! File = " + 
                                          urlobj.path)

                        # Do processing:
                        LOG.info("RDR to SDR processing on npp/viirs with CSPP start!" + 
                                 " Start time = ", start_time)
                        if orbnum:
                            LOG.info("Orb = %d" % orbnum)
                        LOG.info("File = %s" % str(urlobj.path))
                        LOG.info("Cleanup working dir before CSPP start...")
                        cleanup_cspp_workdir(working_dir)
                        # Fix orbit number in RDR file:
                        try:
                            rdr_filename = fix_rdrfile(urlobj.path)
                        except IOError:
                            LOG.error('Failed to fix orbit number in RDR file = ' + str(urlobj.path))
                            import traceback
                            traceback.print_exc(file=sys.stderr)

                        LOG.info("Start CSPP: RDR file = " + str(rdr_filename))
                        run_cspp(rdr_filename)
                        LOG.info("CSPP SDR processing finished...")
                        # Assume everything has gone well! 
                        # Move the files from working dir:
                        result_files = get_files4pps(working_dir)
                        if len(result_files) == 0:
                            LOG.warning("No SDR files available. CSPP probably failed!")
                            continue

                        #start_time = get_datetime(result_files[0])
                        # Use the start time from the RDR message!:
                        #tobj = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%S")
                        tobj = start_time
                        LOG.info("Time used in sub-dir name: " + str(tobj.strftime("%Y-%m-%d %H:%M")))
                        subd = create_pps_subdirname(tobj)
                        LOG.info("Crate sub-directory for sdr files: %s" % str(subd))
                        pack_sdr_files4pps(result_files, subd)
                        make_okay_files(subd)

                        # Now publish:
                        filename = result_files[0]
                        LOG.info("Filename = %s" % filename)
                        to_send = {}
                        # FIXME: Hardcoded machine name ! This is bad !!!
                        to_send['uri'] = ('ssh://safe.smhi.se/' +  
                                          os.path.join(level1_home, 
                                                       filename))
                        to_send['filename'] = filename
                        to_send['instrument'] = 'viirs'
                        to_send['satellite'] = 'NPP'
                        to_send['format'] = 'HDF5'
                        to_send['type'] = 'SDR'
                        to_send['start_time'] = start_time #start_time.isoformat()
                        msg = Message('/oper/polar/direct_readout/norrkoping',
                                      "file", to_send).encode()
                        publisher.send(msg)


    return
import os, sys
_NPP_PREPROC_LOG_FILE = os.environ.get('NPP_PREPROC_LOG_FILE', None)
import logging

if _NPP_PREPROC_LOG_FILE:
    handler = logging.FileHandler(_NPP_PREPROC_LOG_FILE)
else:
    handler = logging.StreamHandler(sys.stderr)

formatter = logging.Formatter(fmt=_DEFAULT_LOG_FORMAT,
                              datefmt=_DEFAULT_TIME_FORMAT)
handler.setFormatter(formatter)

handler.setLevel(10)
LOG.setLevel(10)
LOG.addHandler(handler)



CSPP_ENVS = {"CSPP_HOME": CSPP_HOME,
             "CSPP_REV": "20120215",
             "CSPP_ANC_CACHE_DIR " : os.path.join(CSPP_HOME,'cache'),
             "CSPP_ANC_HOME" : os.path.join(CSPP_HOME,'static'),
             "CSPP_ANC_TILE_PATH" : "%s/static/ADL/data/tiles/Terrain-Eco-ANC-Tile/withMetadata" % (CSPP_HOME),
             "PATH" : '%s/atms/sdr:%s/atms/sdr:%s/viirs/edr:%s/viirs/sdr' % (CSPP_HOME, 
                                                                             CSPP_HOME,
                                                                             CSPP_HOME,
                                                                             CSPP_HOME),
             "ADL_HOME" : "%s/ADL" % (CSPP_HOME),
             "NPP_GRANULE_ID_BASETIME" : "1698019234000000",