Exemple #1
0
def get_pub_address(name, timeout=10, nameserver="localhost"):
    """Get the address of the publisher for a given publisher *name* from the
    nameserver on *nameserver* (localhost by default).
    """

    # Socket to talk to server
    socket = get_context().socket(REQ)
    try:
        socket.setsockopt(LINGER, timeout * 1000)
        socket.connect("tcp://" + nameserver + ":" + str(PORT))
        logger.debug('Connecting to %s',
                     "tcp://" + nameserver + ":" + str(PORT))
        poller = Poller()
        poller.register(socket, POLLIN)

        message = Message("/oper/ns", "request", {"service": name})
        socket.send_string(six.text_type(message))

        # Get the reply.
        sock = poller.poll(timeout=timeout * 1000)
        if sock:
            if sock[0][0] == socket:
                message = Message.decode(socket.recv_string(NOBLOCK))
                return message.data
        else:
            raise TimeoutError("Didn't get an address after %d seconds." %
                               timeout)
    finally:
        socket.close()
Exemple #2
0
    def _check_age(self, pub, min_interval=timedelta(seconds=0)):
        """Check the age of the receiver.
        """
        now = datetime.utcnow()
        if (now - self._last_age_check) <= min_interval:
            return

        LOGGER.debug("%s - checking addresses", str(datetime.utcnow()))
        self._last_age_check = now
        to_del = []
        with self._address_lock:
            for addr, metadata in self._addresses.items():
                atime = metadata["receive_time"]
                if now - atime > self._max_age:
                    mda = {
                        'status': False,
                        'URI': addr,
                        'service': metadata['service']
                    }
                    msg = Message('/address/' + metadata['name'], 'info', mda)
                    to_del.append(addr)
                    LOGGER.info("publish remove '%s'", str(msg))
                    pub.send(msg.encode())
            for addr in to_del:
                del self._addresses[addr]
Exemple #3
0
def get_address(data_type, timeout=2):

    warnings.warn(
        "nameclient.get_address shouldn't be used, " +
        "use posttroll.subscriber.get_address instead...", DeprecationWarning)

    context = zmq.Context()

    # Socket to talk to server
    socket = context.socket(zmq.REQ)
    try:
        socket.connect("tcp://localhost:5555")

        msg = Message("/oper/ns", "request", {"type": data_type})
        socket.send(str(msg))

        # Get the reply.
        poller = zmq.Poller()
        poller.register(socket, zmq.POLLIN)
        s = poller.poll(timeout=timeout * 1000)
        if s:
            if s[0][0] == socket:
                m = Message.decode(socket.recv(zmq.NOBLOCK))
                return m.data
            else:
                raise RuntimeError("Unknown socket ?!?!?")
        else:
            raise TimeoutException("Didn't get an address after %d seconds." %
                                   timeout)
        print "Received reply to ", data_type, ": [", m, "]"
    finally:
        socket.close()
Exemple #4
0
 def test_pub_unicode(self):
     message = Message("/pџтяöll", "info", 'hej')
     with Publish("a_service", 9000) as pub:
         try:
             pub.send(message.encode())
         except UnicodeDecodeError:
             self.fail("Sending raised UnicodeDecodeError unexpectedly!")
Exemple #5
0
    def ack(self, message):
        """Reply with ack to a publication
        """
        for url in gen_dict_extract(message.data, 'uri'):
            uri = urlparse(url)
            pathname = uri.path

            if 'origin' in self._attrs and not fnmatch.fnmatch(
                    os.path.basename(pathname),
                    os.path.basename(globify(self._attrs["origin"]))):
                LOGGER.warning('Client trying to get invalid file: %s',
                               pathname)
                return Message(message.subject,
                               "err",
                               data="{0:s} not reacheable".format(pathname))

            if (self._attrs.get('compression') or self._attrs.get(
                    'delete', 'False').lower() in ["1", "yes", "true", "on"]):
                self._deleter.add(pathname)
        new_msg = Message(message.subject, "ack", data=message.data.copy())
        try:
            new_msg.data['destination'] = clean_url(
                new_msg.data['destination'])
        except KeyError:
            pass
        return new_msg
def request_push(msg, destinations, login):
    # TODO make this a bit more elaborate (like create a new message)
    #hostname = "localhost"
    with cache_lock:
        if msg.data["uid"] in file_cache:
            urlobj = urlparse(msg.data['uri'])
            if socket.gethostbyname(urlobj.netloc) in get_local_ips():
                PUB.send(msg)
            return
        file_cache.append(msg.data["uid"])

    hostname, port = msg.data["request_address"].split(":")
    requester = PushRequester(hostname, int(port))
    req = Message(msg.subject, "push", data=msg.data.copy())

    duris = []
    for destination in destinations:
        duris.append(urlunparse(("ftp",
                                 login + "@" + socket.gethostname(),
                                 destination,
                                 "", "", "")))

    req.data["destinations"] = duris

    LOGGER.info("Requesting: " + str(req))
    response = requester.send_and_recv(req)
    if response.type == "ack":
        LOGGER.debug("Server ack")
    else:
        LOGGER.error("Received an erraneous response from server %s", str(response))
Exemple #7
0
 def run(self):
     while self._loop:
         try:
             socks = dict(self._poller.poll(timeout=2000))
         except ZMQError:
             logger.info("Poller interrupted.")
             continue
         if self._socket in socks and socks[self._socket] == POLLIN:
             logger.debug("Received a request, waiting for the lock")
             with self._lock:
                 message = Message(rawstr=self._socket.recv(NOBLOCK))
                 logger.debug("processing request: " + str(message))
                 reply = Message(subject, "error")
                 try:
                     if message.type == "ping":
                         reply = self.pong()
                     elif (message.type == "request"
                           and message.data["type"] == "scanline"):
                         reply = self.scanline(message)
                     elif (message.type == "notice"
                           and message.data["type"] == "scanline"):
                         reply = self.notice(message)
                     else:  # unknown request
                         reply = self.unknown(message)
                 except:
                     logger.exception("Something went wrong"
                                      " when processing the request:")
                 finally:
                     self.send(reply)
             logger.debug("Lock released from manager")
         else:  # timeout
             pass
Exemple #8
0
    def setUp(self):
        from threading import Lock
        import six.moves.queue as queue
        from posttroll.message import Message

        self.loader = SceneLoader()
        self.prodlist = write_yaml(PRODUCT_LIST)
        self.prodlist_2 = write_yaml(PRODUCT_LIST_TWO_AREAS)
        self.prodlist_2_together = write_yaml(PRODUCT_LIST_TWO_AREAS_TOGETHER)
        self.lock = Lock()
        self.prev_lock = Lock()
        self.output_queue = queue.Queue()
        self.context = {
            'lock': self.lock,
            'prev_lock': self.prev_lock,
            'output_queue': self.output_queue
        }
        self.topic = '/topic'
        self.file_msg = Message(self.topic, 'file', METADATA_FILE)
        self.dataset_msg = Message(self.topic, 'dataset', METADATA_DATASET)
        self.collection_msg = Message(self.topic, 'collection',
                                      METADATA_COLLECTION)
        self.collection_dataset_msg = Message(self.topic, 'collection',
                                              METADATA_COLLECTION_DATASET)
        self.dset1 = Mock(name='dset1')
        self.dset2 = Mock(name='dset2')
        self.scene = Mock(attrs=METADATA_FILE,
                          datasets=[self.dset1, self.dset2])
Exemple #9
0
 def test_encode(self):
     """Test the encoding of a message.
     """
     subject = "/test/whatup/doc"
     atype = "info"
     data = "not much to say"
     msg1 = Message(subject, atype, data=data)
     sender = "%s@%s" % (msg1.user, msg1.host)
     self.assertEquals(
         _MAGICK
         + subject
         + " "
         + atype
         + " "
         + sender
         + " "
         + str(msg1.time.isoformat())
         + " "
         + msg1.version
         + " "
         + "text/ascii"
         + " "
         + data,
         msg1.encode(),
     )
Exemple #10
0
    def push(self, message):
        """Reply to push request
        """
        uri = urlparse(message.data["uri"])
        pathname = uri.path

        if not fnmatch.fnmatch(
                os.path.basename(pathname),
                os.path.basename(globify(self._attrs["origin"]))):
            LOGGER.warning('Client trying to get invalid file: %s', pathname)
            return Message(message.subject,
                           "err",
                           data="{0:s} not reachable".format(pathname))
        try:
            move_it(message, self._attrs)
        except Exception as err:
            return Message(message.subject, "err", data=str(err))
        else:
            if (self._attrs.get('compression') or self._attrs.get(
                    'delete', 'False').lower() in ["1", "yes", "true", "on"]):
                self._deleter.add(pathname)
            new_msg = Message(message.subject,
                              "file",
                              data=message.data.copy())
            new_msg.data['destination'] = clean_url(
                new_msg.data['destination'])
            return new_msg
Exemple #11
0
def get_active_address(name, arec):
    """Get the addresses of the active modules for a given publisher *name*.
    """
    addrs = arec.get(name)
    if addrs:
        return Message("/oper/ns", "info", addrs)
    else:
        return Message("/oper/ns", "info", "")
Exemple #12
0
def _get_cleaned_ack_message(message):
    new_msg = Message(message.subject, "ack", data=message.data.copy())
    try:
        new_msg.data['destination'] = clean_url(new_msg.data['destination'])
    except KeyError:
        pass

    return new_msg
Exemple #13
0
 def test_check_message_okay_message_ok(self):
     """Test for check if message is okay."""
     input_msg = Message.decode(rawstr=TEST_INPUT_MSG)
     result = check_message_okay(input_msg)
     self.assertEqual(result, None)
     input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_DATASET)
     result = check_message_okay(input_msg)
     self.assertEqual(result, None)
Exemple #14
0
    def test_metadata(self):
        """Test metadata encoding/decoding.
        """
        metadata = copy.copy(SOME_METADATA)
        msg = Message.decode(
            Message('/sat/polar/smb/level1', 'file', data=metadata).encode())

        self.assertTrue(msg.data == metadata,
                        msg='Messaging, metadata decoding / encoding failed')
Exemple #15
0
    def test_encode_decode(self):
        """Test the encoding/decoding of the message class.
        """
        msg1 = Message("/test/whatup/doc", "info", data="not much to say")

        sender = "%s@%s" % (msg1.user, msg1.host)
        self.assertTrue(sender == msg1.sender, msg="Messaging, decoding user, host from sender failed")
        msg2 = Message.decode(msg1.encode())
        self.assertTrue(str(msg2) == str(msg1), msg="Messaging, encoding, decoding failed")
Exemple #16
0
    def test_pub_unicode(self):
        from posttroll.message import Message
        from posttroll.publisher import Publish

        message = Message("/pџтяöll", "info", 'hej')
        with Publish("a_service", 9000) as pub:
            try:
                pub.send(message.encode())
            except UnicodeDecodeError:
                self.fail("Sending raised UnicodeDecodeError unexpectedly!")
Exemple #17
0
def _sanitize_message_destination(message):
    sanitized_message = Message(rawstr=str(message))
    try:
        _ = urlparse(message.data['destination'])
    except (KeyError, TypeError):
        pass
    else:
        sanitized_message.data['destination'] = clean_url(
            message.data['destination'])
    return sanitized_message
Exemple #18
0
def get_address(data_type, gc):
    """Get the address of the module for a given *data_type*.
    """
    if data_type == "":
        return Message("/oper/ns", "info", gc.get_addresses())
    # a tuple should be returned...
    for addr in gc.get_addresses():
        if data_type in addr["type"]:
            return Message("/oper/ns", "info", addr)
    return Message("/oper/ns", "info", "")
Exemple #19
0
    def push(self, message):
        """Reply to push request."""
        new_msg = self._move_files(message)
        if new_msg is None:
            new_msg = Message(message.subject,
                              _get_push_message_type(message),
                              data=message.data.copy())
            new_msg.data['destination'] = clean_url(
                new_msg.data['destination'])

        return new_msg
Exemple #20
0
    def test_encode_decode(self):
        """Test the encoding/decoding of the message class.
        """
        msg1 = Message('/test/whatup/doc', 'info', data='not much to say')

        sender = '%s@%s' % (msg1.user, msg1.host)
        self.assertTrue(sender == msg1.sender,
                        msg='Messaging, decoding user, host from sender failed')
        msg2 = Message.decode(msg1.encode())
        self.assertTrue(str(msg2) == str(msg1),
                        msg='Messaging, encoding, decoding failed')
Exemple #21
0
 def scanline(self, message):
     """Reply to scanline request
     """
     sat = message.data["satellite"]
     key = strp_isoformat(message.data["utctime"])
     try:
         data = self._holder.get_data(sat, key)
     except KeyError:
         resp = Message(subject, "missing")
     else:
         resp = Message(subject, "scanline", data, binary=True)
     return resp
Exemple #22
0
def fetch_files(message, destination):
    """Fetch files from the network"""
    new_message = Message(rawstr=str(message))
    if new_message.type == "dataset":
        for dset in new_message.data["dataset"]:
            dset["uri"] = fetch_file(dset["uri"], destination)
    elif new_message.type == "collection":
        raise NotImplementedError
    else:
        new_message.data["uri"] = fetch_file(new_message.data["uri"],
                                             destination)
    return new_message
Exemple #23
0
 def test_encode(self):
     """Test the encoding of a message.
     """
     subject = '/test/whatup/doc'
     atype = "info"
     data = 'not much to say'
     msg1 = Message(subject, atype, data=data)
     sender = '%s@%s' % (msg1.user, msg1.host)
     self.assertEquals(
         _MAGICK + subject + " " + atype + " " + sender + " " +
         str(msg1.time.isoformat()) + " " + msg1.version + " " +
         'text/ascii' + " " + data, msg1.encode())
Exemple #24
0
    def test_encode_decode(self):
        """Test the encoding/decoding of the message class.
        """
        msg1 = Message('/test/whatup/doc', 'info', data='not much to say')

        sender = '%s@%s' % (msg1.user, msg1.host)
        self.assertTrue(
            sender == msg1.sender,
            msg='Messaging, decoding user, host from sender failed')
        msg2 = Message.decode(msg1.encode())
        self.assertTrue(str(msg2) == str(msg1),
                        msg='Messaging, encoding, decoding failed')
Exemple #25
0
def check_and_publish(datatype, rpc_metadata, publish, heartbeat):
    """Check for new files of type *datatype*, with the given *rpc_metadata*
    and publish them through *publish*.
    """
    stamp_config = DatexLastStamp(datatype)

    def younger_than_stamp_files():
        """Uses glob polling to get new files.
        """
        fdir, fglob = datex_config.get_path(datatype)
        del fglob
        fstamp = stamp_config.get_last_stamp()
        for fname, ftime in _get_file_list(datatype,
                                           time_start=fstamp + TIME_EPSILON):
            if datex_config.distribute(datatype):
                yield os.path.join(fdir, fname)
            stamp_config.update_last_stamp(ftime)

    # Give the publisher a little time to initialize
    # (e.g reconnections from subscribers)
    time.sleep(1)
    logger.info('publisher starting for datatype %s' % datatype)
    if heartbeat:
        last_heartbeat = datetime.now() - timedelta(seconds=heartbeat + 1)
    try:
        while (True):
            if (heartbeat and
                (datetime.now() - last_heartbeat).seconds >= heartbeat):
                # Send a heartbeat
                msg = Message('/hearbeat/' + datatype, 'heartbeat',
                              datetime.utcnow().isoformat())
                logger.info('sending: ' + str(msg))
                try:
                    publish.send(str(msg))
                    last_heartbeat = datetime.now()
                except zmq.ZMQError:
                    logger.exception('publish failed')
            for filedesc in younger_than_stamp_files():
                # Publish new files
                data = copy.copy(rpc_metadata)
                data['uri'] += os.path.basename(filedesc)
                msg = Message('/' + datatype, 'file', data)
                logger.info('sending: ' + str(msg))
                try:
                    publish.send(str(msg))
                except zmq.ZMQError:
                    logger.exception('publish failed')
            time.sleep(TIME_WAKEUP)
    except (KeyboardInterrupt, SystemExit):
        pass
    finally:
        logger.info('publisher stopping')
        publish.close()
def test_prepare_posttroll_message(setup_comm, get_config, gethostname):
    """Test setup the posttroll message."""

    get_config.return_value = CONFIG_EXAMPLE
    gethostname.return_value = "my.host.name"

    myconfigfile = "/my/config/file/path"
    myboarders_file = "/my/shape/file/with/country/boarders"
    mymask_file = "/my/shape/file/with/polygons/to/filter/out"

    afpp = ActiveFiresPostprocessing(myconfigfile, myboarders_file,
                                     mymask_file)

    test_filepath = "/my/geojson/file/path"

    input_msg = Message.decode(rawstr=TEST_MSG)
    res_msg = afpp._generate_output_message(test_filepath, input_msg)

    assert res_msg.data['platform_name'] == 'NOAA-20'
    assert res_msg.data['type'] == 'GEOJSON-filtered'
    assert res_msg.data['format'] == 'geojson'
    assert res_msg.data['product'] == 'afimg'
    assert res_msg.subject == '/VIIRS/L2/Fires/PP/National'
    assert res_msg.data['uri'] == 'ssh://my.host.name//my/geojson/file/path'

    input_msg = Message.decode(rawstr=TEST_MSG)

    fake_region_mask = {
        'attributes': {
            'Kod_omr': '9999',
            'Testomr': 'Some area description'
        }
    }
    res_msg = afpp._generate_output_message(test_filepath,
                                            input_msg,
                                            region=fake_region_mask)

    assert res_msg.subject == '/VIIRS/L2/Fires/PP/Regional/9999'

    msg_str = 'No fire detections for this granule'

    result_messages = afpp._generate_no_fires_messages(input_msg, msg_str)

    for (nat_or_reg, res_msg) in zip(['National', 'Regional'],
                                     result_messages):
        assert res_msg.data['info'] == msg_str
        assert res_msg.subject == '/VIIRS/L2/Fires/PP/' + nat_or_reg
        assert 'type' not in res_msg.data
        assert 'format' not in res_msg.data
        assert 'product' not in res_msg.data
        assert 'uri' not in res_msg.data
Exemple #27
0
    def publish_message(self, mymessage):
        """Publish the message."""
        posttroll_msg = Message(mymessage['header'], mymessage['type'],
                                mymessage['content'])
        msg_to_publish = posttroll_msg.encode()

        manager = Manager()
        publisher_q = manager.Queue()

        pub_thread = PPSPublisher(publisher_q, self.nameservers)
        pub_thread.start()
        LOG.info("Sending: " + str(msg_to_publish))
        publisher_q.put(msg_to_publish)
        pub_thread.stop()
Exemple #28
0
    def test_item_translator(self):
        """Test translating dict items."""
        orig = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.tar", "uri": "/home/user/bla.tar"}'
        )
        dest_dir = '/tmp'
        expected = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/tmp/bla.png"}'
        )

        def dummy_cb(var, k):
            dirname, filename = os.path.split(var[k])
            basename, ext = os.path.splitext(filename)
            if dirname:
                dest = os.path.join(dest_dir, basename + '.png')
            else:
                dest = basename + '.png'
            var[k] = dest
            return var

        expected_dict = Message(rawstr=expected).data
        res = translate_dict_item(Message(rawstr=orig).data, 'uri', dummy_cb)
        res = translate_dict_item(res, 'uid', dummy_cb)
        self.assertDictEqual(expected_dict, res)

        expected = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "dataset": [{"uid": "bla1.png", "uri": "/tmp/bla1.png"}, {"uid": "bla2.png", "uri": "/tmp/bla2.png"}]}'
        )  # noqa

        def dummy_cb(var):
            if not var['uid'].endswith('.tar'):
                return var
            dirname, filename = os.path.split(var.pop('uri'))
            basename, ext = os.path.splitext(filename)
            new_names = [basename + str(i) + '.png' for i in range(1, 3)]
            var.pop('uid')
            var['dataset'] = [
                dict(uid=nn, uri=os.path.join(dest_dir, nn))
                for nn in new_names
            ]
            return var

        expected_dict = Message(rawstr=expected).data
        res = translate_dict(
            Message(rawstr=orig).data, ('uri', 'uid'), dummy_cb)
        self.assertDictEqual(expected_dict, res)
Exemple #29
0
    def publish_callback(msg, *args, **kwargs):
        # save to file_cache
        with cache_lock:
            if msg.data['uid'] in file_registery:
                file_registery[msg.data['uid']].append(msg)
                return

            file_registery[msg.data['uid']] = [msg]
        # transform message
        new_msg = Message(msg.subject, msg.type, msg.data.copy())
        new_msg.data['request_address'] = request_address

        # send onwards
        LOGGER.debug('Sending %s', str(new_msg))
        send(str(new_msg))
Exemple #30
0
    def test_metadata(self):
        """Test metadata encoding/decoding.
        """
        metadata = copy.copy(SOME_METADATA)
        msg = Message.decode(Message("/sat/polar/smb/level1", "file", data=metadata).encode())

        self.assertTrue(msg.data == metadata, msg="Messaging, metadata decoding / encoding failed")
Exemple #31
0
 def _run(self):
     port = broadcast_port
     recv = MulticastReceiver(port).settimeout(2.0)
     self._is_running = True
     with Publish("address_receiver", ["addresses"], self._port) as pub:
         try:
             while self._do_run:
                 try:
                     data, fromaddr = recv()
                     del fromaddr
                 except SocketTimeout:
                     continue
                 msg = Message.decode(data)
                 name = msg.subject.split("/")[1]
                 if(msg.type == 'info' and
                    msg.subject.lower().endswith(self._subject)):
                     addr = msg.data["URI"]
                     metadata = copy.copy(msg.data)
                     metadata["name"] = name
                     if debug:
                         print 'receiving address', addr, name, metadata
                     if addr not in self._addresses:
                         pub.send(str(msg))
                         if debug:
                             print 'publishing address', addr, name, metadata
                     self._add(addr, metadata)
         finally:
             self._is_running = False
             recv.close()
Exemple #32
0
def receive_from_zmq(host,
                     port,
                     station,
                     environment,
                     excluded_platforms,
                     days=1):
    """Receive 2met! messages from zeromq.
    """

    # socket = Subscriber(["tcp://localhost:9331"], ["2met!"])
    sock = GMCSubscriber(host, port)
    msg_rec = MessageReceiver(host, excluded_platforms)

    with Publish("receiver", 0, ["HRPT 0", "PDS", "RDR", "EPS 0"]) as pub:
        for rawmsg in sock.recv():
            # TODO:
            # - Watch for idle time in order to detect a hangout
            LOGGER.debug("receive from 2met! %s", str(rawmsg))
            string = TwoMetMessage(rawmsg)
            to_send = msg_rec.receive(string)
            if to_send is None:
                continue
            subject = "/".join(
                ("", to_send['format'], to_send['data_processing_level'],
                 station, environment, "polar", "direct_readout"))
            msg = Message(subject, "file", to_send).encode()
            LOGGER.debug("publishing %s", str(msg))
            pub.send(msg)
            if days:
                msg_rec.clean_passes(days)
Exemple #33
0
    def fun(orig_pathname):
        """Publish what we have."""
        if not fnmatch.fnmatch(orig_pathname, pattern):
            return
        else:
            LOGGER.debug('We have a match: %s', orig_pathname)

        pathname = unpack(orig_pathname, **attrs)

        info = attrs.get("info", {})
        if info:
            info = dict((elt.strip().split('=') for elt in info.split(";")))
            for infokey, infoval in info.items():
                if "," in infoval:
                    info[infokey] = infoval.split(",")

        info.update(parse(attrs["origin"], orig_pathname))
        info['uri'] = pathname
        info['uid'] = os.path.basename(pathname)
        info['request_address'] = attrs.get(
            "request_address", get_own_ip()) + ":" + attrs["request_port"]
        msg = Message(attrs["topic"], 'file', info)
        publisher.send(str(msg))
        with file_cache_lock:
            file_cache.appendleft(attrs["topic"] + '/' + info["uid"])
        LOGGER.debug("Message sent: " + str(msg))
    def process(self, pathname):
        '''Process the event'''
        # New file created and closed
        logger.debug("processing %s", pathname)
        # parse information and create self.info dict{}
        metadata = self.config.copy()
        success = False
        for parser in self.parsers:
            try:
                metadata.update(parser.parse(pathname))
                success = True
                break
            except ValueError:
                pass
            if not success:
                logger.warning("Could not find a matching pattern for %s",
                               pathname)

        metadata['uri'] = pathname
        metadata['uid'] = os.path.basename(pathname)

        if self.tbus_orbit and "orbit_number" in metadata:
            logger.info("Changing orbit number by -1!")
            metadata["orbit_number"] -= 1

        # replace values with corresponding aliases, if any are given
        if self.aliases:
            for key in metadata:
                if key in self.aliases:
                    metadata[key] = self.aliases[key][str(metadata[key])]

        message = Message(self.topic, 'file', metadata)
        logger.info("Publishing message %s" % str(message))
        self.pub.send(str(message))
Exemple #35
0
 def get(self, timeout=None):
     """Return a published message or None if a timeout has happend.
     """
     if timeout:
         timeout *= 1000.
     self.subscriber.connect(self.destination)
     poller = zmq.Poller()
     poller.register(self.subscriber, zmq.POLLIN)
     try:
         while(True):
             try:
                 ret = poller.poll(timeout=timeout)
                 if ret:
                     if ret[0][0] == self.subscriber:
                         msg = Message.decode(
                             self.subscriber.recv(zmq.NOBLOCK))
                         yield msg
                     else:
                         logger.error("WHAT THE HECK")
                 else:
                     # timeout
                     yield None
             except zmq.ZMQError:
                 logger.exception('recv failed')
     finally:
         poller.unregister(self.subscriber)
         self.subscriber.close()        
Exemple #36
0
    def test_translate_dict_item_to_dataset(self):
        """Test translating dict items to dataset."""
        orig = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.tar", "uri": "/home/user/bla.tar"}'
        )
        expected = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "dataset": [{"uid": "bla1.png", "uri": "/tmp/bla1.png"}, {"uid": "bla2.png", "uri": "/tmp/bla2.png"}]}'
        )  # noqa

        expected_dict = Message(rawstr=expected).data
        res = translate_dict(
            Message(rawstr=orig).data, ('uri', 'uid'),
            dummy_callback_translate_dict_item_to_dataset)
        self.assertDictEqual(expected_dict, res)
Exemple #37
0
            def copy_hook(pathname, dest, val=val, pub=pub):
                fname = os.path.basename(pathname)

                destination = urlunparse((dest.scheme, dest.hostname,
                                          os.path.join(dest.path,
                                                       fname), dest.params,
                                          dest.query, dest.fragment))
                info = val.get("info", "")
                if info:
                    info = dict(
                        (elt.strip().split('=') for elt in info.split(";")))
                    for infokey, infoval in info.items():
                        if "," in infoval:
                            info[infokey] = infoval.split(",")
                else:
                    info = {}
                try:
                    info.update(
                        parse(os.path.basename(val["origin"]),
                              os.path.basename(pathname)))
                except ValueError:
                    info.update(
                        parse(
                            os.path.basename(
                                os.path.splitext(val["origin"])[0]),
                            os.path.basename(pathname)))

                info['uri'] = destination
                info['uid'] = fname
                msg = Message(val["topic"], 'file', info)
                pub.send(str(msg))
                LOGGER.debug("Message sent: %s", str(msg))
Exemple #38
0
def get_pub_address(name, timeout=10, nameserver="localhost"):
    """Get the address of the publisher for a given publisher *name* from the
    nameserver on *nameserver* (localhost by default).
    """

    # Socket to talk to server
    socket = context.socket(REQ)
    try:
        socket.setsockopt(LINGER, timeout * 1000)
        socket.connect("tcp://" + nameserver + ":" + str(PORT))

        poller = Poller()
        poller.register(socket, POLLIN)

        message = Message("/oper/ns", "request", {"service": name})
        socket.send(str(message))

        # Get the reply.
        sock = poller.poll(timeout=timeout * 1000)
        if sock:
            if sock[0][0] == socket:
                message = Message.decode(socket.recv(NOBLOCK))
                return message.data
        else:
            raise TimeoutError("Didn't get an address after %d seconds."
                               % timeout)
    finally:
        socket.close()
Exemple #39
0
def get_address(data_type, timeout=2):

    warnings.warn(
        "nameclient.get_address shouldn't be used, " + "use posttroll.subscriber.get_address instead...",
        DeprecationWarning,
    )

    context = zmq.Context()

    # Socket to talk to server
    socket = context.socket(zmq.REQ)
    try:
        socket.connect("tcp://localhost:5555")

        msg = Message("/oper/ns", "request", {"type": data_type})
        socket.send(str(msg))

        # Get the reply.
        poller = zmq.Poller()
        poller.register(socket, zmq.POLLIN)
        s = poller.poll(timeout=timeout * 1000)
        if s:
            if s[0][0] == socket:
                m = Message.decode(socket.recv(zmq.NOBLOCK))
                return m.data
            else:
                raise RuntimeError("Unknown socket ?!?!?")
        else:
            raise TimeoutException("Didn't get an address after %d seconds." % timeout)
        print "Received reply to ", data_type, ": [", m, "]"
    finally:
        socket.close()
Exemple #40
0
    def run(self, *args):
        """Run the listener and answer to requests.
        """
        del args

        arec = AddressReceiver(max_age=self._max_age)
        arec.start()
        port = PORT

        try:
            self.listener = context.socket(REP)
            self.listener.bind("tcp://*:" + str(port))
            poller = Poller()
            poller.register(self.listener, POLLIN)
            while self.loop:
                socks = dict(poller.poll(1000))
                if socks:
                    if socks.get(self.listener) == POLLIN:
                        msg = self.listener.recv()
                else:
                    continue
                logger.debug("Replying to request: " + str(msg))
                msg = Message.decode(msg)
                self.listener.send_unicode(str(get_active_address(
                    msg.data["service"], arec)))
        except KeyboardInterrupt:
            # Needed to stop the nameserver.
            pass
        finally:
            arec.stop()
            self.listener.close()
Exemple #41
0
    def test_translate_dict_values_file_message(self):
        """Test translating message dictionary values."""
        orig = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/home/user/bla.png"}'
        )
        expected = (
            'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
            '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/tmp/bla.png"}'
        )

        expected_dict = Message(rawstr=expected).data
        res = translate_dict_value(
            Message(rawstr=orig).data, 'uri',
            dummy_callback_translate_dict_values)
        self.assertDictEqual(expected_dict, res)
Exemple #42
0
 def run(self):
     last_hb = datetime.now()
     minutes = 2
     while self._loop:
         if datetime.now() - last_hb > timedelta(minutes=minutes):
             logger.error("No heartbeat from " + str(self._pubaddress))
             last_hb = datetime.now()
             minutes = 1440
         socks = dict(self._poller.poll(2000))
         if (socks and
                 self._subsocket in socks and
                 socks[self._subsocket] == POLLIN):
             message = Message.decode(self._subsocket.recv())
         else:
             continue
         if message.type == "have":
             sat = message.data["satellite"]
             key = strp_isoformat(message.data["timecode"])
             elevation = message.data["elevation"]
             quality = message.data.get("quality", 100)
             data = _MirrorGetter(self._req, sat, key)
             self._holder.add(sat, key, elevation, quality, data)
         if message.type == "heartbeat":
             logger.debug("Got heartbeat from " + str(self._pubaddress)
                          + ": " + str(message))
             self._sched.mirror_next_pass = message.data["next_pass"]
             last_hb = datetime.now()
             minutes = 2
Exemple #43
0
 def test_encode(self):
     """Test the encoding of a message.
     """
     subject = '/test/whatup/doc'
     atype = "info"
     data = 'not much to say'
     msg1 = Message(subject, atype, data=data)
     sender = '%s@%s' % (msg1.user, msg1.host)
     self.assertEqual(_MAGICK +
                      subject + " " +
                      atype + " " +
                      sender + " " +
                      str(msg1.time.isoformat()) + " " +
                      msg1.version + " " +
                      'text/ascii' + " " +
                      data,
                      msg1.encode())
Exemple #44
0
    def test_decode(self):
        """Test the decoding of a message.
        """
        rawstr = (_MAGICK +
                  r'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' +
                  r' text/ascii "what' + r"'" + r's up doc"')
        msg = Message.decode(rawstr)

        self.assertTrue(str(msg) == rawstr,
                        msg='Messaging, decoding of message failed')
Exemple #45
0
    def test_decode(self):
        """Test the decoding of a message.
        """
        rawstr = (
            _MAGICK
            + "/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01"
            + ' application/json "what\'s up doc"'
        )
        msg = Message.decode(rawstr)

        self.assertTrue(str(msg) == rawstr, msg="Messaging, decoding of message failed")
    def _check_age(self, pub, min_interval=timedelta(seconds=0)):
        """Check the age of the receiver.
        """
        now = datetime.utcnow()
        if (now - self._last_age_check) <= min_interval:
            return

        LOGGER.debug("%s - checking addresses", str(datetime.utcnow()) )
        self._last_age_check = now
        with self._address_lock:
            for addr, metadata in self._addresses.items():
                atime = metadata["receive_time"]
                if now - atime > self._max_age:
                    mda = {'status': False,
                           'URI': addr,
                           'service': metadata['service']}
                    msg = Message('/address/' + metadata['name'], 'info', mda)
                    del self._addresses[addr]
                    LOGGER.info("publish remove '%s'", str(msg))
                    pub.send(msg.encode())
Exemple #47
0
    def recv(self, timeout=None):
        """Receive, optionally with *timeout* in seconds.
        """
        if timeout:
            timeout *= 1000.

        for sub in list(self.subscribers) + self._hooks:
            self.poller.register(sub, POLLIN)
        self._loop = True
        try:
            while self._loop:
                sleep(0)
                try:
                    socks = dict(self.poller.poll(timeout=timeout))
                    if socks:
                        for sub in self.subscribers:
                            if sub in socks and socks[sub] == POLLIN:
                                m__ = Message.decode(sub.recv_string(NOBLOCK))
                                if not self._filter or self._filter(m__):
                                    if self._translate:
                                        url = urlsplit(self.sub_addr[sub])
                                        host = url[1].split(":")[0]
                                        m__.sender = (m__.sender.split("@")[0]
                                                      + "@" + host)
                                    yield m__

                        for sub in self._hooks:
                            if sub in socks and socks[sub] == POLLIN:
                                m__ = Message.decode(sub.recv_string(NOBLOCK))
                                self._hooks_cb[sub](m__)
                    else:
                        # timeout
                        yield None
                except ZMQError as err:
                    LOGGER.exception("Receive failed: %s", str(err))
        finally:
            for sub in list(self.subscribers) + self._hooks:
                self.poller.unregister(sub)
Exemple #48
0
    def _run(self):
        """Run the receiver.
        """
        port = broadcast_port
        recv = MulticastReceiver(port).settimeout(2.)
        self._is_running = True
        with Publish("address_receiver", self._port, ["addresses"]) as pub:
            try:
                while self._do_run:
                    try:
                        data, fromaddr = recv()
                        del fromaddr
                    except SocketTimeout:
                        logger.debug("Multicast socket timed out on recv!")
                        continue
                    finally:
                        self._check_age(pub, min_interval=self._max_age / 20)
                        if self._do_heartbeat:
                            pub.heartbeat(min_interval=29)
                    msg = Message.decode(data)
                    name = msg.subject.split("/")[1]
                    if(msg.type == 'info' and
                       msg.subject.lower().startswith(self._subject)):
                        addr = msg.data["URI"]
                        msg.data['status'] = True
                        metadata = copy.copy(msg.data)
                        metadata["name"] = name

                        logger.debug('receiving address ' + str(addr)
                                     + " " + str(name) + " " + str(metadata))
                        if addr not in self._addresses:
                            logger.info("nameserver: publish add '%s'",
                                        str(msg))
                            pub.send(msg.encode())
                        self._add(addr, metadata)
            finally:
                self._is_running = False
                recv.close()
Exemple #49
0
    def recv(self, timeout=None):
        if timeout:
            timeout *= 1000.

        for sub in self.subscribers:
            self.poller.register(sub, zmq.POLLIN)
        self._loop = True
        try:
            while(self._loop):
                try:
                    s = dict(self.poller.poll(timeout=timeout))
                    if s:
                        for sub in self.subscribers:
                            if sub in s and s[sub] == zmq.POLLIN:
                                m__ = Message.decode(sub.recv(zmq.NOBLOCK))
                                if self._translate:
                                    url = urlsplit(self.sub_addr[sub])
                                    host = url[1].split(":")[0]
                                    m__.sender = (m__.sender.split("@")[0]
                                                  + "@" + host)
                                # Only accept pre-defined data types 
                                try:
                                    if (self._data_types and
                                        (m__.data['type'] not in
                                         self._data_types)):
                                        continue
                                except (KeyError, TypeError):
                                    pass

                                yield m__
                    else:
                        # timeout
                        yield None
                except zmq.ZMQError:
                    print >>sys.stderr, 'receive failed'
        finally:
            for sub in self.subscribers:
                self.poller.unregister(sub)
Exemple #50
0
    def recv(self, timeout=None):
        """Receive a message, timeout in seconds.
        """

        if timeout:
            timeout *= 1000

        while self._loop:
            with self._lock:
                if not self._loop:
                    break
                subs = dict(self._poller.poll(timeout=timeout))
                if subs:
                    for sub in self.subscribers:
                        if sub in subs and subs[sub] == POLLIN:
                            msg = Message.decode(sub.recv())
                            if self._translate:
                                url = urlsplit(self.sub_addr[sub])
                                host = url[1].split(":")[0]
                                msg.sender = msg.sender.split("@")[0] + "@" + host
                            yield msg
                else:
                    yield None
Exemple #51
0
    # a tuple should be returned...
    for addr in gc.get_addresses():
        if data_type in addr["type"]:
            return Message("/oper/ns", "info", addr)
    return Message("/oper/ns", "info", "")

if __name__ == '__main__':
    GC = GenericConnections("")
    GC.start()
    port = 5555

    try:
        context = zmq.Context()
        listener = context.socket(zmq.REP)
        listener.bind("tcp://*:"+str(port))
        while True:
            m = listener.recv()
            print m
            msg = Message.decode(m)
            listener.send_unicode(str(get_address(msg.data["type"], GC)))

    except KeyboardInterrupt:
        # this is needed for the reception to be interruptible
        pass

    finally:
        print "terminating nameserver..."
        GC.stop()
        listener.close()

    def _run(self):
        """Run the receiver.
        """
        port = broadcast_port
        nameservers = []
        if self._multicast_enabled:
            recv = MulticastReceiver(port).settimeout(2.)
            while True:
                try:
                    recv = MulticastReceiver(port).settimeout(2.)
                    LOGGER.info("Receiver initialized.")
                    break
                except IOError as err:
                    if err.errno == errno.ENODEV:
                        LOGGER.error("Receiver initialization failed "
                                     "(no such device). "
                                     "Trying again in %d s",
                                     10)
                        time.sleep(10)
                    else:
                        raise
        else:
            recv = _SimpleReceiver(port)
            nameservers = ["localhost"]

        self._is_running = True
        with Publish("address_receiver", self._port, ["addresses"],
                     nameservers=nameservers) as pub:
            try:
                while self._do_run:
                    try:
                        data, fromaddr = recv()
                        LOGGER.debug("data %s" % data)
                        del fromaddr
                    except SocketTimeout:
                        if self._multicast_enabled:
                            LOGGER.debug("Multicast socket timed out on recv!")
                            continue
                    finally:
                        self._check_age(pub, min_interval=self._max_age / 20)
                        if self._do_heartbeat:
                            pub.heartbeat(min_interval=29)
                    msg = Message.decode(data)
                    name = msg.subject.split("/")[1]
                    if(msg.type == 'info' and
                       msg.subject.lower().startswith(self._subject)):
                        addr = msg.data["URI"]
                        msg.data['status'] = True
                        metadata = copy.copy(msg.data)
                        metadata["name"] = name

                        LOGGER.debug('receiving address %s %s %s', str(addr),
                                     str(name), str(metadata))
                        if addr not in self._addresses:
                            LOGGER.info("nameserver: publish add '%s'",
                                        str(msg))
                            pub.send(msg.encode())
                        self._add(addr, metadata)
            finally:
                self._is_running = False
                recv.close()