示例#1
0
def Run_subscribing_loop(topic, id):
    global Running
    global Exit_Code
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open("127.0.0.1", 5672)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))
        sequence_num = 0
        global Running
        timestamp = 0
        while (True):
            global Exit_Code
            Message_envelope = yield Subscriber.Next(15)
            if (not Message_envelope):
                if (
                        Exit_Code != -1
                ):  # Got None before receiving all messages, it is an error.
                    Exit_Code = -1
                break
            num = Message_envelope.Amqp.Decode_sequence_number()
            sequence_num = sequence_num + 1
            msg_time = Message_envelope.Amqp.Decode_timestamp()
            # To check timestamp of the message is following our rule
            if (timestamp == 0):
                timestamp = msg_time
            else:
                if (msg_time - timestamp == (60 * 60) * 1000000):
                    timestamp = msg_time
                else:
                    print("Message timestamp " + str(msg_time) +
                          " is not the expected value.")
                    Exit_Code = -1
                    break
            if (num % 100 == 0):
                print(str(id) + ": Sequence number is " + str(num))
            # To check sequence number is following our rule
            if (num != sequence_num):
                Exit_Code = -1
                break
            if (sequence_num == Total_Message):
                if (Exit_Code != -1):
                    Exit_Code = 0
                break

        print(
            str(id) + ": Sequence number for the last record is " +
            str(sequence_num))

    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#2
0
def Run_subscribe_loop():
    global Exit_Code
    Publisher = Synapse_publisher()
    yield Publisher.Open(Synapse_host, Synapse_port)
    Topic_name = "a"

    Contests = Contests_type()

    Contest = Contests.Set_contest_element("Contest1")
    Contest.Set_scheduledStart(Basic_shared_timestamp(2001, 1, 3, 1, 2, 3,
                                                      555))
    Contest.Set_contestName(Basic_shared_text("Contest1"))

    yield Publisher.Publish(
        Topic_name, Contests, None, False, None, 0,
        Datetime_to_synapse_timestamp(datetime.datetime(2015, 01, 01, 0, 0)))

    yield Publisher.Close()

    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Message_factory_type())
        yield Subscriber.Open(Synapse_host, Synapse_port, 5)
        yield Subscriber.Subscribe(
            "a ",
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2015, 01, 01, 0, 0)),
                Supply_metadata=True))
示例#3
0
def Run_subscribing_loop(topic, total_messages):
    global Running
    global Exit_Code
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open(Synapse_host, Synapse_port)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(Begin_utc=1, Supply_metadata=True))
        count = 0
        while (True):
            Message_envelope = yield Subscriber.Next(30)
            if Message_envelope is None:
                raise RuntimeError("Subscriber.Next() timeout unexpected")
            count += 1
            if (count % 1000 == 0):
                print("Received " + str(count) + " messages, topic=" + topic)
            if (count >= total_messages):
                print("All messages are received for topic: " + topic)
                break
    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#4
0
def Run_extend_ack_loop(topic, fullmode=True):
    global Running
    global Exit_Code
    global Extend_ack_in_progress
    try:
        sub = Synapse_subscriber(Basic_waypoints_factory(),
                                 Basic_traceRoute_factory())
        yield sub.Open(Synapse_host, Synapse_port)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
        version = sub.Client.server_properties['version']
        info = sub.Client.server_properties['information']
        print("Synapse information:'" + info + "', version:'" + version + "'")
        props = {}
        props["Topics"] = 1
        if (FullMode):
            print("Subscribe to full mode as well as extend_ack mode")
            yield sub.Subscribe(
                topic,
                Synapse_subscription_options(Begin_utc=1,
                                             Supply_metadata=True))
        else:
            print("Subscribe to extend_ack mode only")
        # in this test, we just keep querying the topic related information from server
        yield sub.Subscribe_extended_ack(props)
        count = 0
        fmcount = 0
        while True:
            msg = yield sub.Next(30)
            if (msg is None):
                raise RuntimeError("Could not query topics")
                break
            if (msg.Is_ack):
                if (msg.Amqp.Properties.headers.has_key("Stats")):
                    array = msg.Amqp.Properties.headers['Stats']
                    if (count % 1000 == 0): print("Stats:" + str(array))
                if (msg.Amqp.Properties.headers.has_key("Topics")):
                    topics = msg.Amqp.Properties.headers["Topics"]
                    if (count % 1000 == 0): print("Topics:" + str(topics))
                count += 1
                # in this test, we just keep querying the topic related information from server (Not yield here)
                if not Extend_ack_in_progress:
                    if (not FullMode):
                        yield (Timeout_wrapper(60)(Subscribe_extend_ack))(
                            sub, props)
                    else:
                        (Timeout_wrapper(60)(Subscribe_extend_ack))(sub, props)
                else:
                    print("Extend_ack_in_progress ...")
            else:
                fmcount += 1
                if (fmcount % 10000 == 0):
                    print("Received " + str(fmcount) +
                          " messages(non-ack) in extend_ack subscription")

    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
def TruncateTopic(timestamp):
    global Running;
    global Exit_Code;
    try : 
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Message_factory_type())
        yield Subscriber.Open(Synapse_host, Synapse_port, 15)
        trunc_time = 1470120000000000
        if (timestamp != 0):
            trunc_time = timestamp * 1000000;
        yield Subscriber._DONT_USE_JUST_YET_Set_topic_sparsification("test.topic", 0, trunc_time)
        yield Subscriber.Close();
    except Exception, e :
        print("Error: " + str(e));
        Exit_Code = -1
def SubscribeMsgs(timestamp):
    global Exit_Code;
    global Running;
    try :
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory())
        yield Subscriber.Open(Synapse_host, Synapse_port)
        print "[" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
        start_time = 1470120000000000;
        if (timestamp != 0):
            start_time = timestamp * 1000000;
        yield Subscriber.Subscribe("test.topic", Synapse_subscription_options(Begin_utc = start_time, Supply_metadata = True))
        sequence_num = 0
        timestamp = 0
        span = 1000000;
        while (True) :
            global Exit_Code
            Message_envelope = yield Subscriber.Next(30)
            if (Message_envelope == None):
                print("Waiting message timeout.");
                Exit_Code = -1;
                break;
            num = Message_envelope.Amqp.Decode_sequence_number();
            sequence_num = sequence_num + 1;
            msg_time = Message_envelope.Amqp.Decode_timestamp();
            # To check timestamp of the message is following our rule
            if (timestamp == 0):
                if (msg_time > start_time):
                    print("Expected timestamp " + str(start_time) + " is not received.");
                    Exit_Code = -1;
                    break;
                else:
                    timestamp = msg_time;
            else:
                if (msg_time - timestamp != span):
                    print("Message is not received as published sequence.")
                    Exit_Code = -1;
                    break;
                else:
                    if msg_time == 1470239999000000:
                        break;
                    else:
                        timestamp = msg_time;

            if (sequence_num % 100 == 0):
                print("Total messages received is " + str(sequence_num))
        print("Sequence number for the last record is " + str(sequence_num))

    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#7
0
def Truncate_topic(topic, size, timestamp):
    global Running
    global Exit_Code
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Message_factory_type())
        yield Subscriber.Open(Synapse_host, Synapse_port, 15)
        trunc_time = 1470120000000000
        if (timestamp != 0):
            trunc_time = timestamp * 1000000
        print("Truncate topic " + topic + " at " + str(trunc_time))

        yield Subscriber._DONT_USE_JUST_YET_Set_topic_sparsification(
            topic, size, trunc_time)
        yield Subscriber.Close()
    except Exception, e:
        print("Error: " + str(e))
        Exit_Code = -1
示例#8
0
def Run_subscribing_loop(topic, id):
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open("127.0.0.1", 5672)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))
        sequence_num = Skip_messages_from_start_size
        global Running
        while (True):
            global Exit_Code
            Message_envelope = yield Subscriber.Next()
            num = Message_envelope.Amqp.Decode_sequence_number()
            sequence_num = sequence_num + 1
            if (num % 100 == 0):
                print(str(id) + ": Sequence number is " + str(num))
            if (num != sequence_num):
                Exit_Code = -1
                print("Unexpected sequence number. Wanted: " +
                      str(sequence_num) + ", got: " + str(num))
                break
            if (sequence_num == Total_Message):
                if (Exit_Code != -1):
                    Exit_Code = 0
                break
        print(
            str(id) + ": Sequence number for the last record is " +
            str(sequence_num))
        Running = Running - 1

    except Exception, e:
        print "Error: " + str(e)
示例#9
0
def Run_subscribing_loop(topic, id, span=0):
    global Running
    global Exit_Code
    try:
        if (span == 0):
            span = 50000
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open("127.0.0.1", 5672)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))
        sequence_num = 0
        timestamp = 0
        while (True):
            Message_envelope = yield Subscriber.Next(30)
            if Message_envelope is None:
                raise RuntimeError("Subscriber.Next() timeout unexpected")
            num = Message_envelope.Amqp.Decode_sequence_number()
            sequence_num = sequence_num + 1
            msg_time = Message_envelope.Amqp.Decode_timestamp()
            # To check timestamp of the message is following our rule
            if (timestamp == 0):
                timestamp = msg_time
            else:
                if (msg_time - timestamp == (60 + id) * span):
                    timestamp = msg_time
                else:
                    print("Message timestamp " + str(msg_time) +
                          " is not the expected value.")
                    Exit_Code = -1
                    break
            if (num % 100 == 0):
                print(str(id) + ": Sequence number is " + str(num))
            # To check sequence number is following our rule
            if (num != sequence_num):
                Exit_Code = -1
                break
            if (sequence_num == Total_Message):
                if (Exit_Code != -1):
                    Exit_Code = 0
                break
            Contests = Message_envelope.Message
            Contest = Contests.Get_contest_element("Some_contest_id_key")
            Contest_name = Contest.Get_contestName()
            element = "Some contest name " + str(sequence_num)
            length = len(element)
            msg_length = length
            power = (id % 10) + 1
            for i in range(power):
                msg_length = msg_length * 2
            temp = Contest_name.Get_value()

            # To check if the content length is correct or not.
            if (len(temp) != msg_length):
                Exit_Code = -1
                break
            # To check content is correct or not
            # From the Publisher method, we know the message should like "ElementElementElement ..."
            for i in range(power):
                if (temp.find(element, i * length) != i * length):
                    Exit_Code = -1
                    break
        print(
            str(id) + ": Sequence number for the last record is " +
            str(sequence_num))
    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#10
0
def Run_subscribing_loop(topic, id):
    global Running
    global Exit_Code
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open("127.0.0.1", 5672)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))
        sequence_num = 0
        timestamp = 0
        while (True):
            Message_envelope = yield Subscriber.Next()
            num = Message_envelope.Amqp.Decode_sequence_number()
            sequence_num = sequence_num + 1
            msg_time = Message_envelope.Amqp.Decode_timestamp()
            # To check timestamp of the message is following our rule
            if (timestamp == 0):
                timestamp = msg_time
            else:
                if (msg_time - timestamp == (600 + id) * 1000000):
                    timestamp = msg_time
                else:
                    print("Message timestamp " + str(msg_time) +
                          " is not the expected value.")
                    Exit_Code = -1
                    break
            if (num % 100 == 0):
                print(str(id) + ": Sequence number is " + str(num))
            # To check sequence number is following our rule
            if (num != sequence_num):
                Exit_Code = -1
                break
            if (sequence_num == Total_Message):
                if (Exit_Code != -1):
                    Exit_Code = 0
                break
            # To check messages include different types.
            if (
                    sequence_num % 4 == 1 or sequence_num % 4 == 2
            ):  # See publisher code above, they should have type name "Contests"
                if (Message_envelope.Type_name != "Contests"):
                    Exit_Code = -1
                    print("Message " + str(sequence_num) + " in topic '" +
                          topic + "' should have type name Contests")
                    break
            else:  # See publisher code above, they should have type name "Contest"
                if (Message_envelope.Type_name != "Contest"):
                    Exit_Code = -1
                    print("Message " + str(sequence_num) + " in topic '" +
                          topic + "' should have type name Contest")
                    break
            # To check message include a not None stream_id
            if (
                    sequence_num % 4 == 2 or sequence_num % 4 == 0
            ):  # See publisher code above, they should have stream_id flag ORed.
                if not (Message_envelope.Flags
                        & Message_envelope.HAS_STREAM_ID):
                    Exit_Code = -1
                    print("Message " + str(sequence_num) + " in topic '" +
                          topic + "' does not has streame_id!")
                    break
            else:  # See publisher code above, they should not have stream_id flag ORed.
                if (Message_envelope.Flags & Message_envelope.HAS_STREAM_ID):
                    Exit_Code = -1
                    print("Message " + str(sequence_num) + " in topic '" +
                          topic +
                          "' includes stream_id, that is not expected.")
                    break
        print(
            str(id) + ": Sequence number for the last record is " +
            str(sequence_num))

    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#11
0
def Run_subscribing_loop(topic,
                         start_seq,
                         port,
                         wp=False,
                         realtime=True,
                         expected_timeout=False):
    global Exit_Code
    global Total_Message
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open("127.0.0.1", port)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))

        Subscriber1 = Synapse_subscriber(Basic_waypoints_factory(),
                                         Basic_traceRoute_factory())
        yield Subscriber1.Open("127.0.0.1", 5672)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber1.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))
        sequence_num = start_seq
        timestamp = 0
        count = 0
        diff = -1
        all_seq = []
        all_seq1 = []
        time_out = 30
        if expected_timeout: time_out = 5
        a = Receive_messages(Subscriber, all_seq, Total_Message, wp, time_out,
                             expected_timeout)
        b = Receive_messages(Subscriber1, all_seq1, Total_Message, False,
                             time_out, expected_timeout)
        yield a
        yield b
        """
        If client is subscribing to the mirrored topics in realtime rather than history data, there is no way to ensure
        the order of the messages from both Synapse(source and mirror target) are identical, therefore we will have
        to sort them by sequence number and then compare the result.
        """
        if Realtime or wp:
            all_seq1.sort()
            all_seq.sort()
        if (all_seq1 != all_seq):
            raise Exception("Messages from two servers are different.")
    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#12
0
def Run_subscribing_loop(topic, id, days_ago):
    global Running
    global Exit_Code
    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Basic_traceRoute_factory())
        yield Subscriber.Open("127.0.0.1", 5672, 600)
        print "[" + datetime.datetime.now().strftime(
            "%Y-%m-%d %H:%M:%S") + "] Opened Synapse"

        yield Subscriber.Subscribe(
            topic,
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2016, 1, 1, 0, 0, 0)),
                Supply_metadata=True))
        sequence_num = 0
        global Total_Topic
        timestamp = 0
        while (True):
            Message_envelope = yield Subscriber.Next()
            num = Message_envelope.Amqp.Decode_sequence_number()
            sequence_num = sequence_num + 1
            msg_time = Message_envelope.Amqp.Decode_timestamp()
            # To check timestamp of the message is following our rule
            if (timestamp == 0):
                timestamp = msg_time
                current = datetime.datetime.utcnow()
                temp = Datetime_to_synapse_timestamp(current)
                temp = temp - (days_ago - 1) * 24 * 60 * 60 * 1000000
                if (timestamp > temp):
                    print("Message timestamp is not as expected " +
                          str(days_ago) + " ago.")
                    Exit_Code = -1
                    break
            else:
                if (msg_time - timestamp == (60 + (id % Total_Topic)) *
                        1000000):
                    timestamp = msg_time
                else:
                    print("Message timestamp " + str(msg_time) +
                          " is not the expected value.")
                    Exit_Code = -1
                    break
            if (num % 100 == 0):
                print(str(id) + ": Sequence number is " + str(num))
            # To check sequence number is following our rule
            if (num != sequence_num):
                Exit_Code = -1
                break
            if (sequence_num == Total_Message):
                if (Exit_Code != -1):
                    Exit_Code = 0
                break
        print(
            str(id) + ": Sequence number for the last record is " +
            str(sequence_num))

    except Exception, e:
        print "Error: " + str(e)
        Exit_Code = -1
示例#13
0
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2015, 01, 01, 0, 0)),
                Supply_metadata=True))
        while (True):
            Message_envelope = yield Subscriber.Next(15)
            if Message_envelope is None:
                break

        Exit_code = -1

    except Exception, e:
        print "Error: " + str(e)
        print "Expected indeed..."

    try:
        Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
                                        Message_factory_type())
        yield Subscriber.Open(Synapse_host, Synapse_port, 5)
        yield Subscriber.Subscribe(
            "a ",
            Synapse_subscription_options(
                Begin_utc=Datetime_to_synapse_timestamp(
                    datetime.datetime(2015, 01, 01, 0, 0)),
                Supply_metadata=True))
        while (True):
            Message_envelope = yield Subscriber.Next()
            break

    except Exception, e:
        print "Error: " + str(e)
        print "Unexpected..."
        Exit_code = -1