def create_listener_notifier(attrs, publisher):
request_address = attrs.get("request_address",
get_own_ip()) + ":" + attrs["request_port"]
delay = float(attrs.get('delay', 0))
if delay > 0:
def send(msg):
Timer(delay, publisher.send, [msg]).start()
else:
def send(msg):
publisher.send(msg)
def publish_callback(msg, *args, **kwargs):
# save to file_cache
with cache_lock:
if msg.data['uid'] in file_registery:
file_registery[msg.data['uid']].append(msg)
return
file_registery[msg.data['uid']] = [msg]
# transform message
new_msg = Message(msg.subject, msg.type, msg.data.copy())
new_msg.data['request_address'] = request_address
# send onwards
LOGGER.debug('Sending %s', str(new_msg))
send(str(new_msg))
listeners = Listeners(publish_callback, **attrs)
return listeners, foo
def hrpt():
td = trollduction.Trollduction()
td.area_def_names = ['euro4', 'scan', 'scan2', 'scan1']
td.image_output_dir = '/tmp'
td.image_filename_template = '%Y%m%d_%H%M_%(area)_%(composite).%(ending)'
td.product_list = {
'euro4': [
'overview', 'ir108', 'cloudtop', 'vis06', 'night_fog',
'night_overview'
],
'scan': [
'overview', 'ir108', 'cloudtop', 'vis06', 'night_fog',
'night_overview'
],
'scan2': [
'overview', 'ir108', 'cloudtop', 'vis06', 'night_fog',
'night_overview'
],
'scan1': [
'overview', 'ir108', 'cloudtop', 'vis06', 'night_fog',
'night_overview'
]
}
td.production_type = 'hrpt_noaa_l1b'
td.init_listener(['tcp://' + get_own_ip() + ':9000'], ['hrpt_noaa'])
td.start_listener()
td.run_single()
def fun(orig_pathname):
"""Publish what we have."""
if not fnmatch.fnmatch(orig_pathname, pattern):
return
else:
LOGGER.debug('We have a match: %s', orig_pathname)
pathname = unpack(orig_pathname, **attrs)
info = attrs.get("info", {})
if info:
info = dict((elt.strip().split('=') for elt in info.split(";")))
for infokey, infoval in info.items():
if "," in infoval:
info[infokey] = infoval.split(",")
info.update(parse(attrs["origin"], orig_pathname))
info['uri'] = pathname
info['uid'] = os.path.basename(pathname)
info['request_address'] = attrs.get(
"request_address", get_own_ip()) + ":" + attrs["request_port"]
msg = Message(attrs["topic"], 'file', info)
publisher.send(str(msg))
with file_cache_lock:
file_cache.appendleft(attrs["topic"] + '/' + info["uid"])
LOGGER.debug("Message sent: " + str(msg))
def _get_notify_message_info(attrs, orig_pathname, pathname):
info = _collect_attribute_info(attrs)
info.update(parse(attrs["origin"], orig_pathname))
info['uri'] = pathname
info['uid'] = os.path.basename(pathname)
info['request_address'] = attrs.get(
"request_address", get_own_ip()) + ":" + attrs["request_port"]
return info
def __init__(self, filemasks, publish_port=9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
def __init__(self, filemasks, publish_port = 9000):
super(EventHandler, self).__init__()
self.logger = logging.getLogger(__name__)
pub_address = "tcp://" + str(get_own_ip()) + ":" + str(publish_port)
self.pub = Publisher(pub_address)
self.filemasks = filemasks
self.filename = ''
self.info = {}
self.filetype = ''
def hrpt():
td = trollduction.Trollduction()
td.area_def_names = ['euro4', 'scan', 'scan2', 'scan1']
td.image_output_dir = '/tmp'
td.image_filename_template = '%Y%m%d_%H%M_%(area)_%(composite).%(ending)'
td.product_list = {'euro4': ['overview', 'ir108', 'cloudtop', 'vis06', 'night_fog', 'night_overview'],
'scan': ['overview', 'ir108', 'cloudtop', 'vis06', 'night_fog', 'night_overview'],
'scan2': ['overview', 'ir108', 'cloudtop', 'vis06', 'night_fog', 'night_overview'],
'scan1': ['overview', 'ir108', 'cloudtop', 'vis06', 'night_fog', 'night_overview']}
td.production_type = 'hrpt_noaa_l1b'
td.init_listener(['tcp://'+get_own_ip()+':9000'], ['hrpt_noaa'])
td.start_listener()
td.run_single()
def _process_message(self, msg):
if _file_already_published(msg):
return
file_registry[msg.data['uid']] = [msg]
request_address = self.ckwargs.get(
"request_address",
get_own_ip()) + ":" + self.ckwargs["request_port"]
delay = float(self.ckwargs.get("delay", 0))
publisher = self.ckwargs["publisher"]
mirror_message = _get_mirror_message(msg, request_address)
if delay:
Timer(delay, publisher.send, [mirror_message]).start()
else:
publish_mirror_message(mirror_message, publisher.send)
def msg():
td = trollduction.Trollduction()
td.image_output_dir = '/tmp'
td.image_filename_template = '%Y%m%d_%H%M_%(area)_%(composite).%(ending)'
td.area_def_names = ['euro4', 'scan2', 'eurol']
td.product_list = {'euro4': ['airmass', 'ash', 'cloudtop', 'convection', 'convection_co2', 'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog', 'night_microphysics', 'night_overview', 'overview', 'red_snow', 'vis06', 'wv_high', 'wv_low'],
'scan2': ['airmass', 'ash', 'cloudtop', 'convection', 'convection_co2', 'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog', 'night_microphysics', 'night_overview', 'overview', 'red_snow', 'vis06', 'wv_high', 'wv_low'],
'met09globeFull': ['airmass', 'ash', 'cloudtop', 'convection', 'convection_co2', 'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog', 'night_microphysics', 'night_overview', 'overview', 'red_snow', 'vis06', 'wv_high', 'wv_low'],
'eurol': ['airmass', 'ash', 'cloudtop', 'convection', 'convection_co2', 'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog', 'night_microphysics', 'night_overview', 'overview', 'red_snow', 'vis06', 'wv_high', 'wv_low'],
'MSGHRVN': ['airmass', 'ash', 'cloudtop', 'convection', 'convection_co2', 'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog', 'night_microphysics', 'night_overview', 'overview', 'red_snow', 'vis06', 'wv_high', 'wv_low']}
td.production_type = 'msg_xrit'
td.init_listener(['tcp://'+get_own_ip()+':9000'], ['hrpt_noaa'])
td.start_listener()
td.run_single()
def run(self):
with Subscribe('', topics=self.attrs['listen'],
addr_listener=True) as sub:
for msg in sub.recv(1):
if msg is None:
if not self.loop:
break
else:
continue
# check that files are local
for uri in gen_dict_extract(msg.data, 'uri'):
urlobj = urlparse(uri)
if (urlobj.scheme not in ['', 'file']
and not socket.gethostbyname(
urlobj.netloc) in get_local_ips()):
break
else:
LOGGER.debug('We have a match: %s', str(msg))
#pathname = unpack(orig_pathname, **attrs)
info = self.attrs.get("info", {})
if info:
info = dict((elt.strip().split('=')
for elt in info.split(";")))
for infokey, infoval in info.items():
if "," in infoval:
info[infokey] = infoval.split(",")
# info.update(parse(attrs["origin"], orig_pathname))
# info['uri'] = pathname
# info['uid'] = os.path.basename(pathname)
info.update(msg.data)
info['request_address'] = self.attrs.get(
"request_address",
get_own_ip()) + ":" + self.attrs["request_port"]
old_data = msg.data
msg = Message(self.attrs["topic"], msg.type, info)
self.publisher.send(str(msg))
with file_cache_lock:
for filename in gen_dict_extract(old_data, 'uid'):
file_cache.appendleft(self.attrs["topic"] + '/' +
filename)
LOGGER.debug("Message sent: " + str(msg))
if not self.loop:
break
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
def msg():
td = trollduction.Trollduction()
td.image_output_dir = '/tmp'
td.image_filename_template = '%Y%m%d_%H%M_%(area)_%(composite).%(ending)'
td.area_def_names = ['euro4', 'scan2', 'eurol']
td.product_list = {
'euro4': [
'airmass', 'ash', 'cloudtop', 'convection', 'convection_co2',
'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog',
'night_microphysics', 'night_overview', 'overview', 'red_snow',
'vis06', 'wv_high', 'wv_low'
],
'scan2': [
'airmass', 'ash', 'cloudtop', 'convection', 'convection_co2',
'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog',
'night_microphysics', 'night_overview', 'overview', 'red_snow',
'vis06', 'wv_high', 'wv_low'
],
'met09globeFull': [
'airmass', 'ash', 'cloudtop', 'convection', 'convection_co2',
'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog',
'night_microphysics', 'night_overview', 'overview', 'red_snow',
'vis06', 'wv_high', 'wv_low'
],
'eurol': [
'airmass', 'ash', 'cloudtop', 'convection', 'convection_co2',
'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog',
'night_microphysics', 'night_overview', 'overview', 'red_snow',
'vis06', 'wv_high', 'wv_low'
],
'MSGHRVN': [
'airmass', 'ash', 'cloudtop', 'convection', 'convection_co2',
'dust', 'fog', 'green_snow', 'ir108', 'natural', 'night_fog',
'night_microphysics', 'night_overview', 'overview', 'red_snow',
'vis06', 'wv_high', 'wv_low'
]
}
td.production_type = 'msg_xrit'
td.init_listener(['tcp://' + get_own_ip() + ':9000'], ['hrpt_noaa'])
td.start_listener()
td.run_single()
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
"""
from datetime import datetime
from posttroll.message import Message
from posttroll.publisher import Publisher, get_own_ip
from posttroll.message_broadcaster import sendaddresstype
import time
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9005"
BROADCASTER = sendaddresstype('dummy_datasource', PUB_ADDRESS, "NWP",
2).start()
PUB = Publisher(PUB_ADDRESS)
oper
test
dev
db
"""
/oper/polar/direct_readout/norrköping
/oper/polar/regional/kangerlusuaq
/oper/geo/0deg
/oper/geo/rss
/oper/geo/iodc
def _collect_message_info(self, msg):
info = _collect_attribute_info(self.attrs)
info.update(msg.data)
info['request_address'] = self.attrs.get(
"request_address", get_own_ip()) + ":" + self.attrs["request_port"]
return info
from posttroll.publisher import Publisher, get_own_ip
from posttroll.subscriber import Subscriber
from posttroll.message import Message
from multiprocessing import Pool
from multiprocessing import Pipe
from collections import deque
from mpop.utils import debug_on
debug_on()
#from multiprocessing import Process
from threading import Thread as Process
import time
OWN_PORT = 9000
OWN_ADDRESS = str(get_own_ip())
#OWN_ADDRESS = str("127.0.0.1")
JOONAS_PORT = 9000
JOONAS_ADDRESS = "10.240.23.37"
class Listener(object):
'''PyTroll listener class for reading messages and adding them to
deque
'''
def __init__(self, ip=None, port=None, processes=2, pipe=None):
'''
'''
self.address_list = []
self.type_list = []
from posttroll.publisher import Publisher, get_own_ip
from posttroll.subscriber import Subscriber
from posttroll.message import Message
from multiprocessing import Pool
from multiprocessing import Pipe
from collections import deque
from mpop.utils import debug_on
debug_on()
#from multiprocessing import Process
from threading import Thread as Process
import time
OWN_PORT = 9000
OWN_ADDRESS = str(get_own_ip())
#OWN_ADDRESS = str("127.0.0.1")
JOONAS_PORT = 9000
JOONAS_ADDRESS = "10.240.23.37"
class Listener(object):
'''PyTroll listener class for reading messages and adding them to
deque
'''
def __init__(self, ip=None, port=None, processes=2, pipe=None):
'''
'''
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
"""
from datetime import datetime
from posttroll.message import Message
from posttroll.publisher import Publisher, get_own_ip
from posttroll.message_broadcaster import sendaddresstype
import time
PUB_ADDRESS = "tcp://" + str(get_own_ip()) + ":9005"
BROADCASTER = sendaddresstype('dummy_datasource', PUB_ADDRESS, "NWP", 2).start()
PUB = Publisher(PUB_ADDRESS)
oper
test
dev
db
"""
/oper/polar/direct_readout/norrköping
/oper/polar/regional/kangerlusuaq
/oper/geo/0deg
/oper/geo/rss
/oper/geo/iodc