def Run_subscribing_loop(topic, id):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
global Running
timestamp = 0
while (True):
global Exit_Code
Message_envelope = yield Subscriber.Next(15)
if (not Message_envelope):
if (
Exit_Code != -1
): # Got None before receiving all messages, it is an error.
Exit_Code = -1
break
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
else:
if (msg_time - timestamp == (60 * 60) * 1000000):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribe_loop():
global Exit_Code
Publisher = Synapse_publisher()
yield Publisher.Open(Synapse_host, Synapse_port)
Topic_name = "a"
Contests = Contests_type()
Contest = Contests.Set_contest_element("Contest1")
Contest.Set_scheduledStart(Basic_shared_timestamp(2001, 1, 3, 1, 2, 3,
555))
Contest.Set_contestName(Basic_shared_text("Contest1"))
yield Publisher.Publish(
Topic_name, Contests, None, False, None, 0,
Datetime_to_synapse_timestamp(datetime.datetime(2015, 01, 01, 0, 0)))
yield Publisher.Close()
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Message_factory_type())
yield Subscriber.Open(Synapse_host, Synapse_port, 5)
yield Subscriber.Subscribe(
"a ",
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2015, 01, 01, 0, 0)),
Supply_metadata=True))
def Run_subscribing_loop(topic, total_messages):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open(Synapse_host, Synapse_port)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(Begin_utc=1, Supply_metadata=True))
count = 0
while (True):
Message_envelope = yield Subscriber.Next(30)
if Message_envelope is None:
raise RuntimeError("Subscriber.Next() timeout unexpected")
count += 1
if (count % 1000 == 0):
print("Received " + str(count) + " messages, topic=" + topic)
if (count >= total_messages):
print("All messages are received for topic: " + topic)
break
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_extend_ack_loop(topic, fullmode=True):
global Running
global Exit_Code
global Extend_ack_in_progress
try:
sub = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield sub.Open(Synapse_host, Synapse_port)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
version = sub.Client.server_properties['version']
info = sub.Client.server_properties['information']
print("Synapse information:'" + info + "', version:'" + version + "'")
props = {}
props["Topics"] = 1
if (FullMode):
print("Subscribe to full mode as well as extend_ack mode")
yield sub.Subscribe(
topic,
Synapse_subscription_options(Begin_utc=1,
Supply_metadata=True))
else:
print("Subscribe to extend_ack mode only")
# in this test, we just keep querying the topic related information from server
yield sub.Subscribe_extended_ack(props)
count = 0
fmcount = 0
while True:
msg = yield sub.Next(30)
if (msg is None):
raise RuntimeError("Could not query topics")
break
if (msg.Is_ack):
if (msg.Amqp.Properties.headers.has_key("Stats")):
array = msg.Amqp.Properties.headers['Stats']
if (count % 1000 == 0): print("Stats:" + str(array))
if (msg.Amqp.Properties.headers.has_key("Topics")):
topics = msg.Amqp.Properties.headers["Topics"]
if (count % 1000 == 0): print("Topics:" + str(topics))
count += 1
# in this test, we just keep querying the topic related information from server (Not yield here)
if not Extend_ack_in_progress:
if (not FullMode):
yield (Timeout_wrapper(60)(Subscribe_extend_ack))(
sub, props)
else:
(Timeout_wrapper(60)(Subscribe_extend_ack))(sub, props)
else:
print("Extend_ack_in_progress ...")
else:
fmcount += 1
if (fmcount % 10000 == 0):
print("Received " + str(fmcount) +
" messages(non-ack) in extend_ack subscription")
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def TruncateTopic(timestamp):
global Running;
global Exit_Code;
try :
Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Message_factory_type())
yield Subscriber.Open(Synapse_host, Synapse_port, 15)
trunc_time = 1470120000000000
if (timestamp != 0):
trunc_time = timestamp * 1000000;
yield Subscriber._DONT_USE_JUST_YET_Set_topic_sparsification("test.topic", 0, trunc_time)
yield Subscriber.Close();
except Exception, e :
print("Error: " + str(e));
Exit_Code = -1
def SubscribeMsgs(timestamp):
global Exit_Code;
global Running;
try :
Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory())
yield Subscriber.Open(Synapse_host, Synapse_port)
print "[" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
start_time = 1470120000000000;
if (timestamp != 0):
start_time = timestamp * 1000000;
yield Subscriber.Subscribe("test.topic", Synapse_subscription_options(Begin_utc = start_time, Supply_metadata = True))
sequence_num = 0
timestamp = 0
span = 1000000;
while (True) :
global Exit_Code
Message_envelope = yield Subscriber.Next(30)
if (Message_envelope == None):
print("Waiting message timeout.");
Exit_Code = -1;
break;
num = Message_envelope.Amqp.Decode_sequence_number();
sequence_num = sequence_num + 1;
msg_time = Message_envelope.Amqp.Decode_timestamp();
# To check timestamp of the message is following our rule
if (timestamp == 0):
if (msg_time > start_time):
print("Expected timestamp " + str(start_time) + " is not received.");
Exit_Code = -1;
break;
else:
timestamp = msg_time;
else:
if (msg_time - timestamp != span):
print("Message is not received as published sequence.")
Exit_Code = -1;
break;
else:
if msg_time == 1470239999000000:
break;
else:
timestamp = msg_time;
if (sequence_num % 100 == 0):
print("Total messages received is " + str(sequence_num))
print("Sequence number for the last record is " + str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Truncate_topic(topic, size, timestamp):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Message_factory_type())
yield Subscriber.Open(Synapse_host, Synapse_port, 15)
trunc_time = 1470120000000000
if (timestamp != 0):
trunc_time = timestamp * 1000000
print("Truncate topic " + topic + " at " + str(trunc_time))
yield Subscriber._DONT_USE_JUST_YET_Set_topic_sparsification(
topic, size, trunc_time)
yield Subscriber.Close()
except Exception, e:
print("Error: " + str(e))
Exit_Code = -1
def Run_subscribing_loop(topic, id):
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = Skip_messages_from_start_size
global Running
while (True):
global Exit_Code
Message_envelope = yield Subscriber.Next()
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
if (num != sequence_num):
Exit_Code = -1
print("Unexpected sequence number. Wanted: " +
str(sequence_num) + ", got: " + str(num))
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
Running = Running - 1
except Exception, e:
print "Error: " + str(e)
def Run_subscribing_loop(topic, id, span=0):
global Running
global Exit_Code
try:
if (span == 0):
span = 50000
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
timestamp = 0
while (True):
Message_envelope = yield Subscriber.Next(30)
if Message_envelope is None:
raise RuntimeError("Subscriber.Next() timeout unexpected")
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
else:
if (msg_time - timestamp == (60 + id) * span):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
Contests = Message_envelope.Message
Contest = Contests.Get_contest_element("Some_contest_id_key")
Contest_name = Contest.Get_contestName()
element = "Some contest name " + str(sequence_num)
length = len(element)
msg_length = length
power = (id % 10) + 1
for i in range(power):
msg_length = msg_length * 2
temp = Contest_name.Get_value()
# To check if the content length is correct or not.
if (len(temp) != msg_length):
Exit_Code = -1
break
# To check content is correct or not
# From the Publisher method, we know the message should like "ElementElementElement ..."
for i in range(power):
if (temp.find(element, i * length) != i * length):
Exit_Code = -1
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic, id):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
timestamp = 0
while (True):
Message_envelope = yield Subscriber.Next()
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
else:
if (msg_time - timestamp == (600 + id) * 1000000):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
# To check messages include different types.
if (
sequence_num % 4 == 1 or sequence_num % 4 == 2
): # See publisher code above, they should have type name "Contests"
if (Message_envelope.Type_name != "Contests"):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic + "' should have type name Contests")
break
else: # See publisher code above, they should have type name "Contest"
if (Message_envelope.Type_name != "Contest"):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic + "' should have type name Contest")
break
# To check message include a not None stream_id
if (
sequence_num % 4 == 2 or sequence_num % 4 == 0
): # See publisher code above, they should have stream_id flag ORed.
if not (Message_envelope.Flags
& Message_envelope.HAS_STREAM_ID):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic + "' does not has streame_id!")
break
else: # See publisher code above, they should not have stream_id flag ORed.
if (Message_envelope.Flags & Message_envelope.HAS_STREAM_ID):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic +
"' includes stream_id, that is not expected.")
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic,
start_seq,
port,
wp=False,
realtime=True,
expected_timeout=False):
global Exit_Code
global Total_Message
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", port)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
Subscriber1 = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber1.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber1.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = start_seq
timestamp = 0
count = 0
diff = -1
all_seq = []
all_seq1 = []
time_out = 30
if expected_timeout: time_out = 5
a = Receive_messages(Subscriber, all_seq, Total_Message, wp, time_out,
expected_timeout)
b = Receive_messages(Subscriber1, all_seq1, Total_Message, False,
time_out, expected_timeout)
yield a
yield b
"""
If client is subscribing to the mirrored topics in realtime rather than history data, there is no way to ensure
the order of the messages from both Synapse(source and mirror target) are identical, therefore we will have
to sort them by sequence number and then compare the result.
"""
if Realtime or wp:
all_seq1.sort()
all_seq.sort()
if (all_seq1 != all_seq):
raise Exception("Messages from two servers are different.")
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic, id, days_ago):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672, 600)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
global Total_Topic
timestamp = 0
while (True):
Message_envelope = yield Subscriber.Next()
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
current = datetime.datetime.utcnow()
temp = Datetime_to_synapse_timestamp(current)
temp = temp - (days_ago - 1) * 24 * 60 * 60 * 1000000
if (timestamp > temp):
print("Message timestamp is not as expected " +
str(days_ago) + " ago.")
Exit_Code = -1
break
else:
if (msg_time - timestamp == (60 + (id % Total_Topic)) *
1000000):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2015, 01, 01, 0, 0)),
Supply_metadata=True))
while (True):
Message_envelope = yield Subscriber.Next(15)
if Message_envelope is None:
break
Exit_code = -1
except Exception, e:
print "Error: " + str(e)
print "Expected indeed..."
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Message_factory_type())
yield Subscriber.Open(Synapse_host, Synapse_port, 5)
yield Subscriber.Subscribe(
"a ",
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2015, 01, 01, 0, 0)),
Supply_metadata=True))
while (True):
Message_envelope = yield Subscriber.Next()
break
except Exception, e:
print "Error: " + str(e)
print "Unexpected..."
Exit_code = -1