def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
"%Y%m%d%H%M%S").isoformat(),
"time_of_last_scanline": datetime.strptime(base[32:46],
"%Y%m%d%H%M%S").isoformat()}
import pprint
pprint.pprint(metadata)
yield Message('/dc/polar/gds', 'file', metadata)
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000"
BROADCASTER = sendaddresstype('p1', PUB_ADDRESS, "HRPT 1b", 2).start()
time.sleep(10)
PUB = Publisher(PUB_ADDRESS)
try:
#for msg in SUB(timeout=1):
# print "Consumer got", msg
counter = 0
while True:
counter += 1
for i in send_new_files():
print "publishing " + str(i)
PUB.send(str(i))
time.sleep(60)
except KeyboardInterrupt:
print "terminating datasource..."
BROADCASTER.stop()
PUB.stop()
oper
test
dev
db
"""
/oper/polar/direct_readout/norrköping
/oper/polar/regional/kangerlusuaq
/oper/geo/0deg
/oper/geo/rss
/oper/geo/iodc
/oper/geo/iodc
"""
try:
counter = 0
while True:
counter += 1
msg = Message('/oper/nwp', 'file', {
"type": "NWP",
"source": "hirlam",
"timestamp": str(datetime.utcnow())
})
print "publishing " + str(msg)
PUB.send(str(msg))
time.sleep(5)
except KeyboardInterrupt:
print "terminating datasource..."
BROADCASTER.stop()
PUB.stop()
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filetypes, publish_port = 9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filetypes = filetypes
self.filename = ''
self.info = {}
self.filetype = ''
self.subject = '/NewFileArrived/'
def __clean__(self):
self.filename = ''
self.filetype = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" %event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.filename = event.name
self.info = {"uri": self.filename,
"satellite": "noaa",
"number": "19",
"instrument": "avhrr"}
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" %str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filemasks, publish_port = 9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
# self.subject = '/NewFileArrived'
def __clean__(self):
self.filename = ''
self.filetype = ''
self.info = ''
self.filepath = ''
self.fullname = ''
self.satellite = ''
self.satnumber = ''
self.year = ''
self.month = ''
self.day = ''
self.hour = ''
self.minute = ''
self.instrument = ''
self.orbit = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" %event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.parse_file_info(event)
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" %str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
self.logger.debug("new file created and closed %s" %event.pathname)
#print event.__dict__
# New file created and closed
if not event.dir:
# parse information and create self.info dict{}
self.parse_file_info(event)
# self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message foo %s" %str(message)
self.pub.send(str(message))
self.__clean__()
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
def parse_file_info(self, event):
'''
'''
self.filename = event.name
self.filepath = event.path
self.fullname = event.pathname
if 'MSG' in self.filename:
# MSG
# H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__
self.filetype = 'msg_xrit'
self.satellite = 'meteosat'
self.instrument = 'seviri'
parts = self.filename.split('-')
self.satnumber = str(int(parts[2].strip('_')[-1])+7)
self.channel = parts[4].strip('_')
self.segment = parts[5].strip('_')
self.year = int(parts[6][:4])
self.month = int(parts[6][4:6])
self.day = int(parts[6][6:8])
self.hour = int(parts[6][8:10])
self.minute = int(parts[6][10:])
self.orbit = ''
if parts[7].strip('_') == 'C':
self.compressed = 1
else:
self.compressed = 0
self.subject = '/NewFileArrived/' + self.filetype
self.info = {"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute,
"segment": self.segment,
"channel": self.channel,
"compressed": self.compressed}
elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename:
# HRPT NOAA l1b file
#
self.filetype = 'hrpt_noaa_l1b'
parts = event.name.split('.')[0].split('_')
self.satellite = parts[1][:4]
self.satnumber = parts[1][4:]
self.year = int(parts[2][:4])
self.month = int(parts[2][4:6])
self.day = int(parts[2][6:])
self.hour = int(parts[3][:2])
self.minute = int(parts[3][2:])
self.instrument = 'avhrr'
self.orbit = parts[4]
self.subject = '/NewFileArrived/' + self.filetype
self.info = {"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute}
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filetypes, publish_port=9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filetypes = filetypes
self.filename = ''
self.info = {}
self.filetype = ''
self.subject = '/NewFileArrived/'
def __clean__(self):
self.filename = ''
self.filetype = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" % event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.filename = event.name
self.info = {
"uri": self.filename,
"satellite": "noaa",
"number": "19",
"instrument": "avhrr"
}
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" % str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filemasks, publish_port=9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
# self.subject = '/NewFileArrived'
def __clean__(self):
self.filename = ''
self.filetype = ''
self.info = ''
self.filepath = ''
self.fullname = ''
self.satellite = ''
self.satnumber = ''
self.year = ''
self.month = ''
self.day = ''
self.hour = ''
self.minute = ''
self.instrument = ''
self.orbit = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" % event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.parse_file_info(event)
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" % str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
self.logger.debug("new file created and closed %s" % event.pathname)
#print event.__dict__
# New file created and closed
if not event.dir:
# parse information and create self.info dict{}
self.parse_file_info(event)
# self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message foo %s" % str(message)
self.pub.send(str(message))
self.__clean__()
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
def parse_file_info(self, event):
'''
'''
self.filename = event.name
self.filepath = event.path
self.fullname = event.pathname
if 'MSG' in self.filename:
# MSG
# H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__
self.filetype = 'msg_xrit'
self.satellite = 'meteosat'
self.instrument = 'seviri'
parts = self.filename.split('-')
self.satnumber = str(int(parts[2].strip('_')[-1]) + 7)
self.channel = parts[4].strip('_')
self.segment = parts[5].strip('_')
self.year = int(parts[6][:4])
self.month = int(parts[6][4:6])
self.day = int(parts[6][6:8])
self.hour = int(parts[6][8:10])
self.minute = int(parts[6][10:])
self.orbit = ''
if parts[7].strip('_') == 'C':
self.compressed = 1
else:
self.compressed = 0
self.subject = '/NewFileArrived/' + self.filetype
self.info = {
"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute,
"segment": self.segment,
"channel": self.channel,
"compressed": self.compressed
}
elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename:
# HRPT NOAA l1b file
#
self.filetype = 'hrpt_noaa_l1b'
parts = event.name.split('.')[0].split('_')
self.satellite = parts[1][:4]
self.satnumber = parts[1][4:]
self.year = int(parts[2][:4])
self.month = int(parts[2][4:6])
self.day = int(parts[2][6:])
self.hour = int(parts[3][:2])
self.minute = int(parts[3][2:])
self.instrument = 'avhrr'
self.orbit = parts[4]
self.subject = '/NewFileArrived/' + self.filetype
self.info = {
"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute
}
PUB = Publisher(PUB_ADDRESS)
oper
test
dev
db
"""
/oper/polar/direct_readout/norrköping
/oper/polar/regional/kangerlusuaq
/oper/geo/0deg
/oper/geo/rss
/oper/geo/iodc
/oper/geo/iodc
"""
try:
counter = 0
while True:
counter += 1
msg = Message('/oper/nwp', 'file', {"type": "NWP",
"source": "hirlam",
"timestamp": str(datetime.utcnow())})
print "publishing " + str(msg)
PUB.send(str(msg))
time.sleep(5)
except KeyboardInterrupt:
print "terminating datasource..."
BROADCASTER.stop()
PUB.stop()
"time_of_first_scanline": datetime.strptime(base[16:30], "%Y%m%d%H%M%S").isoformat(),
"time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat(),
}
import pprint
pprint.pprint(metadata)
yield Message("/dc/polar/gds", "file", metadata)
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000"
BROADCASTER = sendaddresstype("p1", PUB_ADDRESS, "HRPT 1b", 2).start()
time.sleep(10)
PUB = Publisher(PUB_ADDRESS)
try:
# for msg in SUB(timeout=1):
# print "Consumer got", msg
counter = 0
while True:
counter += 1
for i in send_new_files():
print "publishing " + str(i)
PUB.send(str(i))
time.sleep(60)
except KeyboardInterrupt:
print "terminating datasource..."
BROADCASTER.stop()
PUB.stop()
