Example #1
0
    def __init__(self, filemasks, publish_port=9000):
        super(EventHandler, self).__init__()

        self.logger = logging.getLogger(__name__)
        pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
        self.pub = Publisher(pub_address)
        self.filemasks = filemasks
        self.filename = ''
        self.info = {}
        self.filetype = ''
Example #2
0
class MoveItServer(MoveItBase):
    """Wrapper class for Trollmoves Server."""

    def __init__(self, cmd_args):
        """Initialize server."""
        super(MoveItServer, self).__init__(cmd_args, "server")
        LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
        self.sync_publisher = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")

    def run(self):
        """Start the transfer chains."""
        signal.signal(signal.SIGTERM, self.chains_stop)
        signal.signal(signal.SIGHUP, self.signal_reload_cfg_file)
        self.notifier.start()
        self.running = True
        while self.running:
            time.sleep(1)
            self.sync_publisher.heartbeat(30)
Example #3
0
 def __init__(self, filemasks, publish_port = 9000):
     super(EventHandler, self).__init__()
     
     self.logger = logging.getLogger(__name__)
     pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
     self.pub = Publisher(pub_address)
     self.filemasks = filemasks
     self.filename = ''
     self.info = {}
     self.filetype = ''
Example #4
0
    def test_pub_suber(self):
        """Test publisher and subscriber.
        """

        pub_address = "tcp://" + str(get_own_ip()) + ":0"
        pub = Publisher(pub_address)
        addr = pub_address[:-1] + str(pub.port_number)
        sub = Subscriber([addr], '/counter')
        tested = False
        for counter in range(5):
            message = Message("/counter", "info", str(counter))
            pub.send(str(message))
            time.sleep(1)

            msg = sub.recv(2).next()
            if msg is not None:
                self.assertEquals(str(msg), str(message))
                tested = True
        self.assertTrue(tested)
        pub.stop()
Example #5
0
 def __setstate__(self, kwargs):
     """Set things running even when loading from YAML."""
     LOG.debug('Starting publisher')
     self.port = kwargs.get('port', 0)
     self.nameservers = kwargs.get('nameservers', "")
     if self.nameservers is None:
         self.pub = Publisher("tcp://*:" + str(self.port), "l2processor")
     else:
         self.pub = NoisyPublisher('l2processor',
                                   port=self.port,
                                   nameservers=self.nameservers)
         self.pub.start()
Example #6
0
    def test_pub_suber(self):
        """Test publisher and subscriber.
        """

        pub_address = "tcp://" + str(get_own_ip()) + ":0"
        pub = Publisher(pub_address)
        addr = pub_address[:-1] + str(pub.port_number)
        sub = Subscriber([addr], '/counter')
        tested = False
        for counter in range(5):
            message = Message("/counter", "info", str(counter))
            pub.send(str(message))
            time.sleep(1)

            msg = sub.recv(2).next()
            if msg is not None:
                self.assertEquals(str(msg), str(message))
                tested = True
        self.assertTrue(tested)
        pub.stop()
Example #7
0
    else:
        fh = logging.StreamHandler()

    formatter = logging.Formatter(log_format)
    fh.setFormatter(formatter)

    LOGGER.addHandler(fh)
    LOGGER = logging.getLogger('move_it_server')

    pyinotify.log.handlers = [fh]

    LOGGER.info("Starting up.")

    LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))

    PUB = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")

    mask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO
            | pyinotify.IN_CREATE)
    watchman = pyinotify.WatchManager()

    def reload_cfg_file(filename, disable_backlog=False):
        return reload_config(filename,
                             chains,
                             publisher=PUB,
                             disable_backlog=disable_backlog)

    notifier = pyinotify.ThreadedNotifier(
        watchman,
        EventHandler(reload_cfg_file, cmd_filename=cmd_args.config_file))
    watchman.add_watch(os.path.dirname(cmd_args.config_file), mask)
Example #8
0
class EventHandler(ProcessEvent):
    """
    Event handler class for inotify.
    """
    def __init__(self, filetypes, publish_port = 9000):
        super(EventHandler, self).__init__()
        
        self.logger = logging.getLogger(__name__)
        pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
        self.pub = Publisher(pub_address)
        self.filetypes = filetypes
        self.filename = ''
        self.info = {}
        self.filetype = ''
        self.subject = '/NewFileArrived/'
        
    def __clean__(self):
        self.filename = ''
        self.filetype = ''
    
    """
    def is_config_changed(self):
        if os.stat(self.config_file).st_mtime > self.config_mtime:
            return True
    
    def update_config_mtime(self):
        self.config_mtime = os.stat(self.config_file).st_mtime
    
    """
        
    def process_IN_CREATE(self, event):
        """
        When new file is created, publish message
        """
        self.logger.debug("new file created %s" %event.pathname)
        #print event.__dict__
        # New file created
        if not event.dir:
            """            
            if self.is_config_changed():
                self.update_config_mtime()
                self.parse_config()
            """
            self.filename = event.name
            self.info = {"uri": self.filename,
                         "satellite": "noaa",
                         "number": "19",
                         "instrument": "avhrr"}
            self.identify_filetype()
            if self.filetype != '':
                message = self.create_message()            
                print "Publishing message %s" %str(message)
                self.pub.send(str(message))
                self.__clean__()    
    
    
    def process_IN_CLOSE_WRITE(self, event):
        """
        """
                
    def identify_filetype(self):
        """
        """
        for filetype in self.filetypes:
            if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
                self.filetype = filetype
                break
        
    def create_message(self):
        """
        Create broadcasted message
        """
        return Message(self.subject, str(self.filetype), self.info)
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
"""

from datetime import datetime
from posttroll.message import Message
from posttroll.publisher import Publisher, get_own_ip
from posttroll.message_broadcaster import sendaddresstype
import time

PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9005"
BROADCASTER = sendaddresstype('dummy_datasource', PUB_ADDRESS, "NWP", 2).start()

PUB = Publisher(PUB_ADDRESS)

oper
test
dev
db

"""
/oper/polar/direct_readout/norrköping
/oper/polar/regional/kangerlusuaq
/oper/geo/0deg
/oper/geo/rss
/oper/geo/iodc
/oper/geo/iodc
"""
Example #10
0
class EventHandler(ProcessEvent):
    """
    Event handler class for inotify.
    """
    def __init__(self, filemasks, publish_port = 9000):
        super(EventHandler, self).__init__()
        
        self.logger = logging.getLogger(__name__)
        pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
        self.pub = Publisher(pub_address)
        self.filemasks = filemasks
        self.filename = ''
        self.info = {}
        self.filetype = ''
#        self.subject = '/NewFileArrived'
        
    def __clean__(self):
        self.filename = ''
        self.filetype = ''
        self.info = ''
        self.filepath = ''
        self.fullname = ''
        self.satellite = ''
        self.satnumber = ''
        self.year = ''
        self.month = ''
        self.day = ''
        self.hour = ''
        self.minute = ''
        self.instrument = ''
        self.orbit = ''
    
    """
    def is_config_changed(self):
        if os.stat(self.config_file).st_mtime > self.config_mtime:
            return True
    
    def update_config_mtime(self):
        self.config_mtime = os.stat(self.config_file).st_mtime
    
    """
        
    def process_IN_CREATE(self, event):
        """
        When new file is created, publish message
        """
        self.logger.debug("new file created %s" %event.pathname)
        #print event.__dict__
        # New file created
        if not event.dir:
            """            
            if self.is_config_changed():
                self.update_config_mtime()
                self.parse_config()
            """
            self.parse_file_info(event)
            self.identify_filetype()
            if self.filetype != '':
                message = self.create_message()            
                print "Publishing message %s" %str(message)
                self.pub.send(str(message))
                self.__clean__()    
    
    
    def process_IN_CLOSE_WRITE(self, event):
        """
        """
        self.logger.debug("new file created and closed %s" %event.pathname)
        #print event.__dict__
        # New file created and closed
        if not event.dir:
            # parse information and create self.info dict{}
            self.parse_file_info(event)
#            self.identify_filetype()
            if self.filetype != '':
                message = self.create_message()            
                print "Publishing message foo %s" %str(message)
                self.pub.send(str(message))
                self.__clean__()    

                
    def identify_filetype(self):
        """
        """
        for filetype in self.filetypes:
            if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
                self.filetype = filetype
                break
        
    def create_message(self):
        """
        Create broadcasted message
        """
        return Message(self.subject, str(self.filetype), self.info)


    def parse_file_info(self, event):
        '''
        '''
        self.filename = event.name
        self.filepath = event.path
        self.fullname = event.pathname

        if 'MSG' in self.filename:
            # MSG
            # H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__
            self.filetype = 'msg_xrit'
            self.satellite = 'meteosat'
            self.instrument = 'seviri'
            parts = self.filename.split('-')
            self.satnumber = str(int(parts[2].strip('_')[-1])+7)
            self.channel = parts[4].strip('_')
            self.segment = parts[5].strip('_')
            self.year = int(parts[6][:4])
            self.month = int(parts[6][4:6])
            self.day = int(parts[6][6:8])
            self.hour = int(parts[6][8:10])
            self.minute = int(parts[6][10:])
            self.orbit = ''
            if parts[7].strip('_') == 'C':
                self.compressed = 1
            else:
                self.compressed = 0

            self.subject = '/NewFileArrived/' + self.filetype
            self.info = {"uri": self.fullname,
                         "satellite": self.satellite,
                         "satnumber": self.satnumber,
                         "instrument": self.instrument,
                         "orbit": self.orbit,
                         "year": self.year,
                         "month": self.month,
                         "day": self.day,
                         "hour": self.hour,
                         "minute": self.minute,
                         "segment": self.segment,
                         "channel": self.channel,
                         "compressed": self.compressed}

        elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename:
            # HRPT NOAA l1b file
            #
            self.filetype = 'hrpt_noaa_l1b'
            parts = event.name.split('.')[0].split('_')
            self.satellite = parts[1][:4]
            self.satnumber = parts[1][4:]
            self.year = int(parts[2][:4])
            self.month = int(parts[2][4:6])
            self.day = int(parts[2][6:])
            self.hour = int(parts[3][:2])
            self.minute = int(parts[3][2:])
            self.instrument = 'avhrr'
            self.orbit = parts[4]

            self.subject = '/NewFileArrived/' + self.filetype
            self.info = {"uri": self.fullname,
                         "satellite": self.satellite,
                         "satnumber": self.satnumber,
                         "instrument": self.instrument,
                         "orbit": self.orbit,
                         "year": self.year,
                         "month": self.month,
                         "day": self.day,
                         "hour": self.hour,
                         "minute": self.minute}
Example #11
0
 def __init__(self, cmd_args):
     super(MoveItMirror, self).__init__(cmd_args, "mirror")
     self.cache_lock = Lock()
     LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
     self.pub = Publisher("tcp://*:" + str(cmd_args.port), "move_it_mirror")
Example #12
0
class EventHandler(ProcessEvent):
    """
    Event handler class for inotify.
    """
    def __init__(self, filetypes, publish_port=9000):
        super(EventHandler, self).__init__()

        self.logger = logging.getLogger(__name__)
        pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
        self.pub = Publisher(pub_address)
        self.filetypes = filetypes
        self.filename = ''
        self.info = {}
        self.filetype = ''
        self.subject = '/NewFileArrived/'

    def __clean__(self):
        self.filename = ''
        self.filetype = ''

    """
    def is_config_changed(self):
        if os.stat(self.config_file).st_mtime > self.config_mtime:
            return True
    
    def update_config_mtime(self):
        self.config_mtime = os.stat(self.config_file).st_mtime
    
    """

    def process_IN_CREATE(self, event):
        """
        When new file is created, publish message
        """
        self.logger.debug("new file created %s" % event.pathname)
        #print event.__dict__
        # New file created
        if not event.dir:
            """            
            if self.is_config_changed():
                self.update_config_mtime()
                self.parse_config()
            """
            self.filename = event.name
            self.info = {
                "uri": self.filename,
                "satellite": "noaa",
                "number": "19",
                "instrument": "avhrr"
            }
            self.identify_filetype()
            if self.filetype != '':
                message = self.create_message()
                print "Publishing message %s" % str(message)
                self.pub.send(str(message))
                self.__clean__()

    def process_IN_CLOSE_WRITE(self, event):
        """
        """

    def identify_filetype(self):
        """
        """
        for filetype in self.filetypes:
            if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
                self.filetype = filetype
                break

    def create_message(self):
        """
        Create broadcasted message
        """
        return Message(self.subject, str(self.filetype), self.info)
Example #13
0
def create_publisher(port, publisher_name):
    """Create a publisher using port *port*."""
    LOGGER.info("Starting publisher on port %s.", str(port))
    return Publisher("tcp://*:" + str(port), publisher_name)
Example #14
0
class EventHandler(ProcessEvent):
    """
    Event handler class for inotify.
    """
    def __init__(self, filemasks, publish_port=9000):
        super(EventHandler, self).__init__()

        self.logger = logging.getLogger(__name__)
        pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
        self.pub = Publisher(pub_address)
        self.filemasks = filemasks
        self.filename = ''
        self.info = {}
        self.filetype = ''


#        self.subject = '/NewFileArrived'

    def __clean__(self):
        self.filename = ''
        self.filetype = ''
        self.info = ''
        self.filepath = ''
        self.fullname = ''
        self.satellite = ''
        self.satnumber = ''
        self.year = ''
        self.month = ''
        self.day = ''
        self.hour = ''
        self.minute = ''
        self.instrument = ''
        self.orbit = ''

    """
    def is_config_changed(self):
        if os.stat(self.config_file).st_mtime > self.config_mtime:
            return True
    
    def update_config_mtime(self):
        self.config_mtime = os.stat(self.config_file).st_mtime
    
    """

    def process_IN_CREATE(self, event):
        """
        When new file is created, publish message
        """
        self.logger.debug("new file created %s" % event.pathname)
        #print event.__dict__
        # New file created
        if not event.dir:
            """            
            if self.is_config_changed():
                self.update_config_mtime()
                self.parse_config()
            """
            self.parse_file_info(event)
            self.identify_filetype()
            if self.filetype != '':
                message = self.create_message()
                print "Publishing message %s" % str(message)
                self.pub.send(str(message))
                self.__clean__()

    def process_IN_CLOSE_WRITE(self, event):
        """
        """
        self.logger.debug("new file created and closed %s" % event.pathname)
        #print event.__dict__
        # New file created and closed
        if not event.dir:
            # parse information and create self.info dict{}
            self.parse_file_info(event)
            #            self.identify_filetype()
            if self.filetype != '':
                message = self.create_message()
                print "Publishing message foo %s" % str(message)
                self.pub.send(str(message))
                self.__clean__()

    def identify_filetype(self):
        """
        """
        for filetype in self.filetypes:
            if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
                self.filetype = filetype
                break

    def create_message(self):
        """
        Create broadcasted message
        """
        return Message(self.subject, str(self.filetype), self.info)

    def parse_file_info(self, event):
        '''
        '''
        self.filename = event.name
        self.filepath = event.path
        self.fullname = event.pathname

        if 'MSG' in self.filename:
            # MSG
            # H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__
            self.filetype = 'msg_xrit'
            self.satellite = 'meteosat'
            self.instrument = 'seviri'
            parts = self.filename.split('-')
            self.satnumber = str(int(parts[2].strip('_')[-1]) + 7)
            self.channel = parts[4].strip('_')
            self.segment = parts[5].strip('_')
            self.year = int(parts[6][:4])
            self.month = int(parts[6][4:6])
            self.day = int(parts[6][6:8])
            self.hour = int(parts[6][8:10])
            self.minute = int(parts[6][10:])
            self.orbit = ''
            if parts[7].strip('_') == 'C':
                self.compressed = 1
            else:
                self.compressed = 0

            self.subject = '/NewFileArrived/' + self.filetype
            self.info = {
                "uri": self.fullname,
                "satellite": self.satellite,
                "satnumber": self.satnumber,
                "instrument": self.instrument,
                "orbit": self.orbit,
                "year": self.year,
                "month": self.month,
                "day": self.day,
                "hour": self.hour,
                "minute": self.minute,
                "segment": self.segment,
                "channel": self.channel,
                "compressed": self.compressed
            }

        elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename:
            # HRPT NOAA l1b file
            #
            self.filetype = 'hrpt_noaa_l1b'
            parts = event.name.split('.')[0].split('_')
            self.satellite = parts[1][:4]
            self.satnumber = parts[1][4:]
            self.year = int(parts[2][:4])
            self.month = int(parts[2][4:6])
            self.day = int(parts[2][6:])
            self.hour = int(parts[3][:2])
            self.minute = int(parts[3][2:])
            self.instrument = 'avhrr'
            self.orbit = parts[4]

            self.subject = '/NewFileArrived/' + self.filetype
            self.info = {
                "uri": self.fullname,
                "satellite": self.satellite,
                "satnumber": self.satnumber,
                "instrument": self.instrument,
                "orbit": self.orbit,
                "year": self.year,
                "month": self.month,
                "day": self.day,
                "hour": self.hour,
                "minute": self.minute
            }
Example #15
0
 def __init__(self, cmd_args):
     """Initialize server."""
     super(MoveItServer, self).__init__(cmd_args, "server")
     LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
     self.sync_publisher = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")
Example #16
0
            "time_of_first_scanline": datetime.strptime(base[16:30],
                                                        "%Y%m%d%H%M%S").isoformat(),
            "time_of_last_scanline": datetime.strptime(base[32:46],
                                                        "%Y%m%d%H%M%S").isoformat()}
        import pprint
        pprint.pprint(metadata)
        yield Message('/dc/polar/gds', 'file', metadata)



PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000"
BROADCASTER = sendaddresstype('p1', PUB_ADDRESS, "HRPT 1b", 2).start()

time.sleep(10)

PUB = Publisher(PUB_ADDRESS)

try:
    #for msg in SUB(timeout=1):
    #    print "Consumer got", msg
    counter = 0
    while True:
        counter += 1
        for i in send_new_files():
            print "publishing " + str(i)
            PUB.send(str(i))
        time.sleep(60)
except KeyboardInterrupt:
    print "terminating datasource..."
    BROADCASTER.stop()
    PUB.stop()
Example #17
0
            "format": "EPS 1b",
            "time_of_first_scanline": datetime.strptime(base[16:30], "%Y%m%d%H%M%S").isoformat(),
            "time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat(),
        }
        import pprint

        pprint.pprint(metadata)
        yield Message("/dc/polar/gds", "file", metadata)


PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000"
BROADCASTER = sendaddresstype("p1", PUB_ADDRESS, "HRPT 1b", 2).start()

time.sleep(10)

PUB = Publisher(PUB_ADDRESS)

try:
    # for msg in SUB(timeout=1):
    #    print "Consumer got", msg
    counter = 0
    while True:
        counter += 1
        for i in send_new_files():
            print "publishing " + str(i)
            PUB.send(str(i))
        time.sleep(60)
except KeyboardInterrupt:
    print "terminating datasource..."
    BROADCASTER.stop()
    PUB.stop()