def __init__(self, filemasks, publish_port=9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filemasks = filemasks self.filename = '' self.info = {} self.filetype = ''
class MoveItServer(MoveItBase): """Wrapper class for Trollmoves Server.""" def __init__(self, cmd_args): """Initialize server.""" super(MoveItServer, self).__init__(cmd_args, "server") LOGGER.info("Starting publisher on port %s.", str(cmd_args.port)) self.sync_publisher = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server") def run(self): """Start the transfer chains.""" signal.signal(signal.SIGTERM, self.chains_stop) signal.signal(signal.SIGHUP, self.signal_reload_cfg_file) self.notifier.start() self.running = True while self.running: time.sleep(1) self.sync_publisher.heartbeat(30)
def __init__(self, filemasks, publish_port = 9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filemasks = filemasks self.filename = '' self.info = {} self.filetype = ''
def test_pub_suber(self): """Test publisher and subscriber. """ pub_address = "tcp://" + str(get_own_ip()) + ":0" pub = Publisher(pub_address) addr = pub_address[:-1] + str(pub.port_number) sub = Subscriber([addr], '/counter') tested = False for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = sub.recv(2).next() if msg is not None: self.assertEquals(str(msg), str(message)) tested = True self.assertTrue(tested) pub.stop()
def __setstate__(self, kwargs): """Set things running even when loading from YAML.""" LOG.debug('Starting publisher') self.port = kwargs.get('port', 0) self.nameservers = kwargs.get('nameservers', "") if self.nameservers is None: self.pub = Publisher("tcp://*:" + str(self.port), "l2processor") else: self.pub = NoisyPublisher('l2processor', port=self.port, nameservers=self.nameservers) self.pub.start()
else: fh = logging.StreamHandler() formatter = logging.Formatter(log_format) fh.setFormatter(formatter) LOGGER.addHandler(fh) LOGGER = logging.getLogger('move_it_server') pyinotify.log.handlers = [fh] LOGGER.info("Starting up.") LOGGER.info("Starting publisher on port %s.", str(cmd_args.port)) PUB = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server") mask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_CREATE) watchman = pyinotify.WatchManager() def reload_cfg_file(filename, disable_backlog=False): return reload_config(filename, chains, publisher=PUB, disable_backlog=disable_backlog) notifier = pyinotify.ThreadedNotifier( watchman, EventHandler(reload_cfg_file, cmd_filename=cmd_args.config_file)) watchman.add_watch(os.path.dirname(cmd_args.config_file), mask)
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filetypes, publish_port = 9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filetypes = filetypes self.filename = '' self.info = {} self.filetype = '' self.subject = '/NewFileArrived/' def __clean__(self): self.filename = '' self.filetype = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" %event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.filename = event.name self.info = {"uri": self.filename, "satellite": "noaa", "number": "19", "instrument": "avhrr"} self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" %str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info)
# You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ """ from datetime import datetime from posttroll.message import Message from posttroll.publisher import Publisher, get_own_ip from posttroll.message_broadcaster import sendaddresstype import time PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9005" BROADCASTER = sendaddresstype('dummy_datasource', PUB_ADDRESS, "NWP", 2).start() PUB = Publisher(PUB_ADDRESS) oper test dev db """ /oper/polar/direct_readout/norrköping /oper/polar/regional/kangerlusuaq /oper/geo/0deg /oper/geo/rss /oper/geo/iodc /oper/geo/iodc """
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filemasks, publish_port = 9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filemasks = filemasks self.filename = '' self.info = {} self.filetype = '' # self.subject = '/NewFileArrived' def __clean__(self): self.filename = '' self.filetype = '' self.info = '' self.filepath = '' self.fullname = '' self.satellite = '' self.satnumber = '' self.year = '' self.month = '' self.day = '' self.hour = '' self.minute = '' self.instrument = '' self.orbit = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" %event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.parse_file_info(event) self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" %str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ self.logger.debug("new file created and closed %s" %event.pathname) #print event.__dict__ # New file created and closed if not event.dir: # parse information and create self.info dict{} self.parse_file_info(event) # self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message foo %s" %str(message) self.pub.send(str(message)) self.__clean__() def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info) def parse_file_info(self, event): ''' ''' self.filename = event.name self.filepath = event.path self.fullname = event.pathname if 'MSG' in self.filename: # MSG # H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__ self.filetype = 'msg_xrit' self.satellite = 'meteosat' self.instrument = 'seviri' parts = self.filename.split('-') self.satnumber = str(int(parts[2].strip('_')[-1])+7) self.channel = parts[4].strip('_') self.segment = parts[5].strip('_') self.year = int(parts[6][:4]) self.month = int(parts[6][4:6]) self.day = int(parts[6][6:8]) self.hour = int(parts[6][8:10]) self.minute = int(parts[6][10:]) self.orbit = '' if parts[7].strip('_') == 'C': self.compressed = 1 else: self.compressed = 0 self.subject = '/NewFileArrived/' + self.filetype self.info = {"uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute, "segment": self.segment, "channel": self.channel, "compressed": self.compressed} elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename: # HRPT NOAA l1b file # self.filetype = 'hrpt_noaa_l1b' parts = event.name.split('.')[0].split('_') self.satellite = parts[1][:4] self.satnumber = parts[1][4:] self.year = int(parts[2][:4]) self.month = int(parts[2][4:6]) self.day = int(parts[2][6:]) self.hour = int(parts[3][:2]) self.minute = int(parts[3][2:]) self.instrument = 'avhrr' self.orbit = parts[4] self.subject = '/NewFileArrived/' + self.filetype self.info = {"uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute}
def __init__(self, cmd_args): super(MoveItMirror, self).__init__(cmd_args, "mirror") self.cache_lock = Lock() LOGGER.info("Starting publisher on port %s.", str(cmd_args.port)) self.pub = Publisher("tcp://*:" + str(cmd_args.port), "move_it_mirror")
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filetypes, publish_port=9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filetypes = filetypes self.filename = '' self.info = {} self.filetype = '' self.subject = '/NewFileArrived/' def __clean__(self): self.filename = '' self.filetype = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" % event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.filename = event.name self.info = { "uri": self.filename, "satellite": "noaa", "number": "19", "instrument": "avhrr" } self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" % str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info)
def create_publisher(port, publisher_name): """Create a publisher using port *port*.""" LOGGER.info("Starting publisher on port %s.", str(port)) return Publisher("tcp://*:" + str(port), publisher_name)
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filemasks, publish_port=9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filemasks = filemasks self.filename = '' self.info = {} self.filetype = '' # self.subject = '/NewFileArrived' def __clean__(self): self.filename = '' self.filetype = '' self.info = '' self.filepath = '' self.fullname = '' self.satellite = '' self.satnumber = '' self.year = '' self.month = '' self.day = '' self.hour = '' self.minute = '' self.instrument = '' self.orbit = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" % event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.parse_file_info(event) self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" % str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ self.logger.debug("new file created and closed %s" % event.pathname) #print event.__dict__ # New file created and closed if not event.dir: # parse information and create self.info dict{} self.parse_file_info(event) # self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message foo %s" % str(message) self.pub.send(str(message)) self.__clean__() def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info) def parse_file_info(self, event): ''' ''' self.filename = event.name self.filepath = event.path self.fullname = event.pathname if 'MSG' in self.filename: # MSG # H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__ self.filetype = 'msg_xrit' self.satellite = 'meteosat' self.instrument = 'seviri' parts = self.filename.split('-') self.satnumber = str(int(parts[2].strip('_')[-1]) + 7) self.channel = parts[4].strip('_') self.segment = parts[5].strip('_') self.year = int(parts[6][:4]) self.month = int(parts[6][4:6]) self.day = int(parts[6][6:8]) self.hour = int(parts[6][8:10]) self.minute = int(parts[6][10:]) self.orbit = '' if parts[7].strip('_') == 'C': self.compressed = 1 else: self.compressed = 0 self.subject = '/NewFileArrived/' + self.filetype self.info = { "uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute, "segment": self.segment, "channel": self.channel, "compressed": self.compressed } elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename: # HRPT NOAA l1b file # self.filetype = 'hrpt_noaa_l1b' parts = event.name.split('.')[0].split('_') self.satellite = parts[1][:4] self.satnumber = parts[1][4:] self.year = int(parts[2][:4]) self.month = int(parts[2][4:6]) self.day = int(parts[2][6:]) self.hour = int(parts[3][:2]) self.minute = int(parts[3][2:]) self.instrument = 'avhrr' self.orbit = parts[4] self.subject = '/NewFileArrived/' + self.filetype self.info = { "uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute }
def __init__(self, cmd_args): """Initialize server.""" super(MoveItServer, self).__init__(cmd_args, "server") LOGGER.info("Starting publisher on port %s.", str(cmd_args.port)) self.sync_publisher = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")
"time_of_first_scanline": datetime.strptime(base[16:30], "%Y%m%d%H%M%S").isoformat(), "time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat()} import pprint pprint.pprint(metadata) yield Message('/dc/polar/gds', 'file', metadata) PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000" BROADCASTER = sendaddresstype('p1', PUB_ADDRESS, "HRPT 1b", 2).start() time.sleep(10) PUB = Publisher(PUB_ADDRESS) try: #for msg in SUB(timeout=1): # print "Consumer got", msg counter = 0 while True: counter += 1 for i in send_new_files(): print "publishing " + str(i) PUB.send(str(i)) time.sleep(60) except KeyboardInterrupt: print "terminating datasource..." BROADCASTER.stop() PUB.stop()
"format": "EPS 1b", "time_of_first_scanline": datetime.strptime(base[16:30], "%Y%m%d%H%M%S").isoformat(), "time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat(), } import pprint pprint.pprint(metadata) yield Message("/dc/polar/gds", "file", metadata) PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000" BROADCASTER = sendaddresstype("p1", PUB_ADDRESS, "HRPT 1b", 2).start() time.sleep(10) PUB = Publisher(PUB_ADDRESS) try: # for msg in SUB(timeout=1): # print "Consumer got", msg counter = 0 while True: counter += 1 for i in send_new_files(): print "publishing " + str(i) PUB.send(str(i)) time.sleep(60) except KeyboardInterrupt: print "terminating datasource..." BROADCASTER.stop() PUB.stop()