def __init__(self, filemasks, publish_port=9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
class MoveItServer(MoveItBase):
"""Wrapper class for Trollmoves Server."""
def __init__(self, cmd_args):
"""Initialize server."""
super(MoveItServer, self).__init__(cmd_args, "server")
LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
self.sync_publisher = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")
def run(self):
"""Start the transfer chains."""
signal.signal(signal.SIGTERM, self.chains_stop)
signal.signal(signal.SIGHUP, self.signal_reload_cfg_file)
self.notifier.start()
self.running = True
while self.running:
time.sleep(1)
self.sync_publisher.heartbeat(30)
def __init__(self, filemasks, publish_port = 9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
def __setstate__(self, kwargs):
"""Set things running even when loading from YAML."""
LOG.debug('Starting publisher')
self.port = kwargs.get('port', 0)
self.nameservers = kwargs.get('nameservers', "")
if self.nameservers is None:
self.pub = Publisher("tcp://*:" + str(self.port), "l2processor")
else:
self.pub = NoisyPublisher('l2processor',
port=self.port,
nameservers=self.nameservers)
self.pub.start()
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
else:
fh = logging.StreamHandler()
formatter = logging.Formatter(log_format)
fh.setFormatter(formatter)
LOGGER.addHandler(fh)
LOGGER = logging.getLogger('move_it_server')
pyinotify.log.handlers = [fh]
LOGGER.info("Starting up.")
LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
PUB = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")
mask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO
| pyinotify.IN_CREATE)
watchman = pyinotify.WatchManager()
def reload_cfg_file(filename, disable_backlog=False):
return reload_config(filename,
chains,
publisher=PUB,
disable_backlog=disable_backlog)
notifier = pyinotify.ThreadedNotifier(
watchman,
EventHandler(reload_cfg_file, cmd_filename=cmd_args.config_file))
watchman.add_watch(os.path.dirname(cmd_args.config_file), mask)
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filetypes, publish_port = 9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filetypes = filetypes
self.filename = ''
self.info = {}
self.filetype = ''
self.subject = '/NewFileArrived/'
def __clean__(self):
self.filename = ''
self.filetype = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" %event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.filename = event.name
self.info = {"uri": self.filename,
"satellite": "noaa",
"number": "19",
"instrument": "avhrr"}
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" %str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
"""
from datetime import datetime
from posttroll.message import Message
from posttroll.publisher import Publisher, get_own_ip
from posttroll.message_broadcaster import sendaddresstype
import time
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9005"
BROADCASTER = sendaddresstype('dummy_datasource', PUB_ADDRESS, "NWP", 2).start()
PUB = Publisher(PUB_ADDRESS)
oper
test
dev
db
"""
/oper/polar/direct_readout/norrköping
/oper/polar/regional/kangerlusuaq
/oper/geo/0deg
/oper/geo/rss
/oper/geo/iodc
/oper/geo/iodc
"""
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filemasks, publish_port = 9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
# self.subject = '/NewFileArrived'
def __clean__(self):
self.filename = ''
self.filetype = ''
self.info = ''
self.filepath = ''
self.fullname = ''
self.satellite = ''
self.satnumber = ''
self.year = ''
self.month = ''
self.day = ''
self.hour = ''
self.minute = ''
self.instrument = ''
self.orbit = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" %event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.parse_file_info(event)
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" %str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
self.logger.debug("new file created and closed %s" %event.pathname)
#print event.__dict__
# New file created and closed
if not event.dir:
# parse information and create self.info dict{}
self.parse_file_info(event)
# self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message foo %s" %str(message)
self.pub.send(str(message))
self.__clean__()
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
def parse_file_info(self, event):
'''
'''
self.filename = event.name
self.filepath = event.path
self.fullname = event.pathname
if 'MSG' in self.filename:
# MSG
# H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__
self.filetype = 'msg_xrit'
self.satellite = 'meteosat'
self.instrument = 'seviri'
parts = self.filename.split('-')
self.satnumber = str(int(parts[2].strip('_')[-1])+7)
self.channel = parts[4].strip('_')
self.segment = parts[5].strip('_')
self.year = int(parts[6][:4])
self.month = int(parts[6][4:6])
self.day = int(parts[6][6:8])
self.hour = int(parts[6][8:10])
self.minute = int(parts[6][10:])
self.orbit = ''
if parts[7].strip('_') == 'C':
self.compressed = 1
else:
self.compressed = 0
self.subject = '/NewFileArrived/' + self.filetype
self.info = {"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute,
"segment": self.segment,
"channel": self.channel,
"compressed": self.compressed}
elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename:
# HRPT NOAA l1b file
#
self.filetype = 'hrpt_noaa_l1b'
parts = event.name.split('.')[0].split('_')
self.satellite = parts[1][:4]
self.satnumber = parts[1][4:]
self.year = int(parts[2][:4])
self.month = int(parts[2][4:6])
self.day = int(parts[2][6:])
self.hour = int(parts[3][:2])
self.minute = int(parts[3][2:])
self.instrument = 'avhrr'
self.orbit = parts[4]
self.subject = '/NewFileArrived/' + self.filetype
self.info = {"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute}
def __init__(self, cmd_args):
super(MoveItMirror, self).__init__(cmd_args, "mirror")
self.cache_lock = Lock()
LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
self.pub = Publisher("tcp://*:" + str(cmd_args.port), "move_it_mirror")
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filetypes, publish_port=9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filetypes = filetypes
self.filename = ''
self.info = {}
self.filetype = ''
self.subject = '/NewFileArrived/'
def __clean__(self):
self.filename = ''
self.filetype = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" % event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.filename = event.name
self.info = {
"uri": self.filename,
"satellite": "noaa",
"number": "19",
"instrument": "avhrr"
}
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" % str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
def create_publisher(port, publisher_name):
"""Create a publisher using port *port*."""
LOGGER.info("Starting publisher on port %s.", str(port))
return Publisher("tcp://*:" + str(port), publisher_name)
class EventHandler(ProcessEvent):
"""
Event handler class for inotify.
"""
def __init__(self, filemasks, publish_port=9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
# self.subject = '/NewFileArrived'
def __clean__(self):
self.filename = ''
self.filetype = ''
self.info = ''
self.filepath = ''
self.fullname = ''
self.satellite = ''
self.satnumber = ''
self.year = ''
self.month = ''
self.day = ''
self.hour = ''
self.minute = ''
self.instrument = ''
self.orbit = ''
"""
def is_config_changed(self):
if os.stat(self.config_file).st_mtime > self.config_mtime:
return True
def update_config_mtime(self):
self.config_mtime = os.stat(self.config_file).st_mtime
"""
def process_IN_CREATE(self, event):
"""
When new file is created, publish message
"""
self.logger.debug("new file created %s" % event.pathname)
#print event.__dict__
# New file created
if not event.dir:
"""
if self.is_config_changed():
self.update_config_mtime()
self.parse_config()
"""
self.parse_file_info(event)
self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message %s" % str(message)
self.pub.send(str(message))
self.__clean__()
def process_IN_CLOSE_WRITE(self, event):
"""
"""
self.logger.debug("new file created and closed %s" % event.pathname)
#print event.__dict__
# New file created and closed
if not event.dir:
# parse information and create self.info dict{}
self.parse_file_info(event)
# self.identify_filetype()
if self.filetype != '':
message = self.create_message()
print "Publishing message foo %s" % str(message)
self.pub.send(str(message))
self.__clean__()
def identify_filetype(self):
"""
"""
for filetype in self.filetypes:
if fnmatch.fnmatch(self.filename, '*' + filetype + '*'):
self.filetype = filetype
break
def create_message(self):
"""
Create broadcasted message
"""
return Message(self.subject, str(self.filetype), self.info)
def parse_file_info(self, event):
'''
'''
self.filename = event.name
self.filepath = event.path
self.fullname = event.pathname
if 'MSG' in self.filename:
# MSG
# H-000-MSG3__-MSG3________-HRV______-000001___-201402130515-__
self.filetype = 'msg_xrit'
self.satellite = 'meteosat'
self.instrument = 'seviri'
parts = self.filename.split('-')
self.satnumber = str(int(parts[2].strip('_')[-1]) + 7)
self.channel = parts[4].strip('_')
self.segment = parts[5].strip('_')
self.year = int(parts[6][:4])
self.month = int(parts[6][4:6])
self.day = int(parts[6][6:8])
self.hour = int(parts[6][8:10])
self.minute = int(parts[6][10:])
self.orbit = ''
if parts[7].strip('_') == 'C':
self.compressed = 1
else:
self.compressed = 0
self.subject = '/NewFileArrived/' + self.filetype
self.info = {
"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute,
"segment": self.segment,
"channel": self.channel,
"compressed": self.compressed
}
elif 'hrpt' in self.filename and 'noaa' in self.filename and 'l1b' in self.filename:
# HRPT NOAA l1b file
#
self.filetype = 'hrpt_noaa_l1b'
parts = event.name.split('.')[0].split('_')
self.satellite = parts[1][:4]
self.satnumber = parts[1][4:]
self.year = int(parts[2][:4])
self.month = int(parts[2][4:6])
self.day = int(parts[2][6:])
self.hour = int(parts[3][:2])
self.minute = int(parts[3][2:])
self.instrument = 'avhrr'
self.orbit = parts[4]
self.subject = '/NewFileArrived/' + self.filetype
self.info = {
"uri": self.fullname,
"satellite": self.satellite,
"satnumber": self.satnumber,
"instrument": self.instrument,
"orbit": self.orbit,
"year": self.year,
"month": self.month,
"day": self.day,
"hour": self.hour,
"minute": self.minute
}
def __init__(self, cmd_args):
"""Initialize server."""
super(MoveItServer, self).__init__(cmd_args, "server")
LOGGER.info("Starting publisher on port %s.", str(cmd_args.port))
self.sync_publisher = Publisher("tcp://*:" + str(cmd_args.port), "move_it_server")
"time_of_first_scanline": datetime.strptime(base[16:30],
"%Y%m%d%H%M%S").isoformat(),
"time_of_last_scanline": datetime.strptime(base[32:46],
"%Y%m%d%H%M%S").isoformat()}
import pprint
pprint.pprint(metadata)
yield Message('/dc/polar/gds', 'file', metadata)
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000"
BROADCASTER = sendaddresstype('p1', PUB_ADDRESS, "HRPT 1b", 2).start()
time.sleep(10)
PUB = Publisher(PUB_ADDRESS)
try:
#for msg in SUB(timeout=1):
# print "Consumer got", msg
counter = 0
while True:
counter += 1
for i in send_new_files():
print "publishing " + str(i)
PUB.send(str(i))
time.sleep(60)
except KeyboardInterrupt:
print "terminating datasource..."
BROADCASTER.stop()
PUB.stop()
"format": "EPS 1b",
"time_of_first_scanline": datetime.strptime(base[16:30], "%Y%m%d%H%M%S").isoformat(),
"time_of_last_scanline": datetime.strptime(base[32:46], "%Y%m%d%H%M%S").isoformat(),
}
import pprint
pprint.pprint(metadata)
yield Message("/dc/polar/gds", "file", metadata)
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9000"
BROADCASTER = sendaddresstype("p1", PUB_ADDRESS, "HRPT 1b", 2).start()
time.sleep(10)
PUB = Publisher(PUB_ADDRESS)
try:
# for msg in SUB(timeout=1):
# print "Consumer got", msg
counter = 0
while True:
counter += 1
for i in send_new_files():
print "publishing " + str(i)
PUB.send(str(i))
time.sleep(60)
except KeyboardInterrupt:
print "terminating datasource..."
BROADCASTER.stop()
PUB.stop()
