def test_pub_suber(self): """Test publisher and subscriber. """ pub_address = "tcp://" + str(get_own_ip()) + ":0" pub = Publisher(pub_address) addr = pub_address[:-1] + str(pub.port_number) sub = Subscriber([addr], '/counter') tested = False for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = sub.recv(2).next() if msg is not None: self.assertEquals(str(msg), str(message)) tested = True self.assertTrue(tested) pub.stop()
"%Y%m%d%H%M%S").isoformat(), "time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat()} import pprint pprint.pprint(metadata) yield Message('/dc/polar/gds', 'file', metadata) PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000" BROADCASTER = sendaddresstype('p1', PUB_ADDRESS, "HRPT 1b", 2).start() time.sleep(10) PUB = Publisher(PUB_ADDRESS) try: #for msg in SUB(timeout=1): # print "Consumer got", msg counter = 0 while True: counter += 1 for i in send_new_files(): print "publishing " + str(i) PUB.send(str(i)) time.sleep(60) except KeyboardInterrupt: print "terminating datasource..." BROADCASTER.stop() PUB.stop()
oper test dev db """ /oper/polar/direct_readout/norrköping /oper/polar/regional/kangerlusuaq /oper/geo/0deg /oper/geo/rss /oper/geo/iodc /oper/geo/iodc """ try: counter = 0 while True: counter += 1 msg = Message('/oper/nwp', 'file', { "type": "NWP", "source": "hirlam", "timestamp": str(datetime.utcnow()) }) print "publishing " + str(msg) PUB.send(str(msg)) time.sleep(5) except KeyboardInterrupt: print "terminating datasource..." BROADCASTER.stop() PUB.stop()
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filetypes, publish_port = 9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filetypes = filetypes self.filename = '' self.info = {} self.filetype = '' self.subject = '/NewFileArrived/' def __clean__(self): self.filename = '' self.filetype = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" %event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.filename = event.name self.info = {"uri": self.filename, "satellite": "noaa", "number": "19", "instrument": "avhrr"} self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" %str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info)
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filemasks, publish_port = 9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filemasks = filemasks self.filename = '' self.info = {} self.filetype = '' # self.subject = '/NewFileArrived' def __clean__(self): self.filename = '' self.filetype = '' self.info = '' self.filepath = '' self.fullname = '' self.satellite = '' self.satnumber = '' self.year = '' self.month = '' self.day = '' self.hour = '' self.minute = '' self.instrument = '' self.orbit = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" %event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.parse_file_info(event) self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" %str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ self.logger.debug("new file created and closed %s" %event.pathname) #print event.__dict__ # New file created and closed if not event.dir: # parse information and create self.info dict{} self.parse_file_info(event) # self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message foo %s" %str(message) self.pub.send(str(message)) self.__clean__() def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info) def parse_file_info(self, event): ''' ''' self.filename = event.name self.filepath = event.path self.fullname = event.pathname if 'MSG' in self.filename: # MSG # H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__ self.filetype = 'msg_xrit' self.satellite = 'meteosat' self.instrument = 'seviri' parts = self.filename.split('-') self.satnumber = str(int(parts[2].strip('_')[-1])+7) self.channel = parts[4].strip('_') self.segment = parts[5].strip('_') self.year = int(parts[6][:4]) self.month = int(parts[6][4:6]) self.day = int(parts[6][6:8]) self.hour = int(parts[6][8:10]) self.minute = int(parts[6][10:]) self.orbit = '' if parts[7].strip('_') == 'C': self.compressed = 1 else: self.compressed = 0 self.subject = '/NewFileArrived/' + self.filetype self.info = {"uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute, "segment": self.segment, "channel": self.channel, "compressed": self.compressed} elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename: # HRPT NOAA l1b file # self.filetype = 'hrpt_noaa_l1b' parts = event.name.split('.')[0].split('_') self.satellite = parts[1][:4] self.satnumber = parts[1][4:] self.year = int(parts[2][:4]) self.month = int(parts[2][4:6]) self.day = int(parts[2][6:]) self.hour = int(parts[3][:2]) self.minute = int(parts[3][2:]) self.instrument = 'avhrr' self.orbit = parts[4] self.subject = '/NewFileArrived/' + self.filetype self.info = {"uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute}
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filetypes, publish_port=9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filetypes = filetypes self.filename = '' self.info = {} self.filetype = '' self.subject = '/NewFileArrived/' def __clean__(self): self.filename = '' self.filetype = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" % event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.filename = event.name self.info = { "uri": self.filename, "satellite": "noaa", "number": "19", "instrument": "avhrr" } self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" % str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info)
class EventHandler(ProcessEvent): """ Event handler class for inotify. """ def __init__(self, filemasks, publish_port=9000): super(EventHandler, self).__init__() self.logger = logging.getLogger(__name__) pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port) self.pub = Publisher(pub_address) self.filemasks = filemasks self.filename = '' self.info = {} self.filetype = '' # self.subject = '/NewFileArrived' def __clean__(self): self.filename = '' self.filetype = '' self.info = '' self.filepath = '' self.fullname = '' self.satellite = '' self.satnumber = '' self.year = '' self.month = '' self.day = '' self.hour = '' self.minute = '' self.instrument = '' self.orbit = '' """ def is_config_changed(self): if os.stat(self.config_file).st_mtime > self.config_mtime: return True def update_config_mtime(self): self.config_mtime = os.stat(self.config_file).st_mtime """ def process_IN_CREATE(self, event): """ When new file is created, publish message """ self.logger.debug("new file created %s" % event.pathname) #print event.__dict__ # New file created if not event.dir: """ if self.is_config_changed(): self.update_config_mtime() self.parse_config() """ self.parse_file_info(event) self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message %s" % str(message) self.pub.send(str(message)) self.__clean__() def process_IN_CLOSE_WRITE(self, event): """ """ self.logger.debug("new file created and closed %s" % event.pathname) #print event.__dict__ # New file created and closed if not event.dir: # parse information and create self.info dict{} self.parse_file_info(event) # self.identify_filetype() if self.filetype != '': message = self.create_message() print "Publishing message foo %s" % str(message) self.pub.send(str(message)) self.__clean__() def identify_filetype(self): """ """ for filetype in self.filetypes: if fnmatch.fnmatch(self.filename, '*' + filetype + '*'): self.filetype = filetype break def create_message(self): """ Create broadcasted message """ return Message(self.subject, str(self.filetype), self.info) def parse_file_info(self, event): ''' ''' self.filename = event.name self.filepath = event.path self.fullname = event.pathname if 'MSG' in self.filename: # MSG # H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__ self.filetype = 'msg_xrit' self.satellite = 'meteosat' self.instrument = 'seviri' parts = self.filename.split('-') self.satnumber = str(int(parts[2].strip('_')[-1]) + 7) self.channel = parts[4].strip('_') self.segment = parts[5].strip('_') self.year = int(parts[6][:4]) self.month = int(parts[6][4:6]) self.day = int(parts[6][6:8]) self.hour = int(parts[6][8:10]) self.minute = int(parts[6][10:]) self.orbit = '' if parts[7].strip('_') == 'C': self.compressed = 1 else: self.compressed = 0 self.subject = '/NewFileArrived/' + self.filetype self.info = { "uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute, "segment": self.segment, "channel": self.channel, "compressed": self.compressed } elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename: # HRPT NOAA l1b file # self.filetype = 'hrpt_noaa_l1b' parts = event.name.split('.')[0].split('_') self.satellite = parts[1][:4] self.satnumber = parts[1][4:] self.year = int(parts[2][:4]) self.month = int(parts[2][4:6]) self.day = int(parts[2][6:]) self.hour = int(parts[3][:2]) self.minute = int(parts[3][2:]) self.instrument = 'avhrr' self.orbit = parts[4] self.subject = '/NewFileArrived/' + self.filetype self.info = { "uri": self.fullname, "satellite": self.satellite, "satnumber": self.satnumber, "instrument": self.instrument, "orbit": self.orbit, "year": self.year, "month": self.month, "day": self.day, "hour": self.hour, "minute": self.minute }
PUB = Publisher(PUB_ADDRESS) oper test dev db """ /oper/polar/direct_readout/norrköping /oper/polar/regional/kangerlusuaq /oper/geo/0deg /oper/geo/rss /oper/geo/iodc /oper/geo/iodc """ try: counter = 0 while True: counter += 1 msg = Message('/oper/nwp', 'file', {"type": "NWP", "source": "hirlam", "timestamp": str(datetime.utcnow())}) print "publishing " + str(msg) PUB.send(str(msg)) time.sleep(5) except KeyboardInterrupt: print "terminating datasource..." BROADCASTER.stop() PUB.stop()
"time_of_first_scanline": datetime.strptime(base[16:30], "%Y%m%d%H%M%S").isoformat(), "time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat(), } import pprint pprint.pprint(metadata) yield Message("/dc/polar/gds", "file", metadata) PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000" BROADCASTER = sendaddresstype("p1", PUB_ADDRESS, "HRPT 1b", 2).start() time.sleep(10) PUB = Publisher(PUB_ADDRESS) try: # for msg in SUB(timeout=1): # print "Consumer got", msg counter = 0 while True: counter += 1 for i in send_new_files(): print "publishing " + str(i) PUB.send(str(i)) time.sleep(60) except KeyboardInterrupt: print "terminating datasource..." BROADCASTER.stop() PUB.stop()