def Run_subscribing_loop(topic, total_messages):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open(Synapse_host, Synapse_port)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(Begin_utc=1, Supply_metadata=True))
count = 0
while (True):
Message_envelope = yield Subscriber.Next(30)
if Message_envelope is None:
raise RuntimeError("Subscriber.Next() timeout unexpected")
count += 1
if (count % 1000 == 0):
print("Received " + str(count) + " messages, topic=" + topic)
if (count >= total_messages):
print("All messages are received for topic: " + topic)
break
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic, id):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
global Running
timestamp = 0
while (True):
global Exit_Code
Message_envelope = yield Subscriber.Next(15)
if (not Message_envelope):
if (
Exit_Code != -1
): # Got None before receiving all messages, it is an error.
Exit_Code = -1
break
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
else:
if (msg_time - timestamp == (60 * 60) * 1000000):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_extend_ack_loop(topic, fullmode=True):
global Running
global Exit_Code
global Extend_ack_in_progress
try:
sub = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield sub.Open(Synapse_host, Synapse_port)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
version = sub.Client.server_properties['version']
info = sub.Client.server_properties['information']
print("Synapse information:'" + info + "', version:'" + version + "'")
props = {}
props["Topics"] = 1
if (FullMode):
print("Subscribe to full mode as well as extend_ack mode")
yield sub.Subscribe(
topic,
Synapse_subscription_options(Begin_utc=1,
Supply_metadata=True))
else:
print("Subscribe to extend_ack mode only")
# in this test, we just keep querying the topic related information from server
yield sub.Subscribe_extended_ack(props)
count = 0
fmcount = 0
while True:
msg = yield sub.Next(30)
if (msg is None):
raise RuntimeError("Could not query topics")
break
if (msg.Is_ack):
if (msg.Amqp.Properties.headers.has_key("Stats")):
array = msg.Amqp.Properties.headers['Stats']
if (count % 1000 == 0): print("Stats:" + str(array))
if (msg.Amqp.Properties.headers.has_key("Topics")):
topics = msg.Amqp.Properties.headers["Topics"]
if (count % 1000 == 0): print("Topics:" + str(topics))
count += 1
# in this test, we just keep querying the topic related information from server (Not yield here)
if not Extend_ack_in_progress:
if (not FullMode):
yield (Timeout_wrapper(60)(Subscribe_extend_ack))(
sub, props)
else:
(Timeout_wrapper(60)(Subscribe_extend_ack))(sub, props)
else:
print("Extend_ack_in_progress ...")
else:
fmcount += 1
if (fmcount % 10000 == 0):
print("Received " + str(fmcount) +
" messages(non-ack) in extend_ack subscription")
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def SubscribeMsgs(timestamp):
global Exit_Code;
global Running;
try :
Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory())
yield Subscriber.Open(Synapse_host, Synapse_port)
print "[" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
start_time = 1470120000000000;
if (timestamp != 0):
start_time = timestamp * 1000000;
yield Subscriber.Subscribe("test.topic", Synapse_subscription_options(Begin_utc = start_time, Supply_metadata = True))
sequence_num = 0
timestamp = 0
span = 1000000;
while (True) :
global Exit_Code
Message_envelope = yield Subscriber.Next(30)
if (Message_envelope == None):
print("Waiting message timeout.");
Exit_Code = -1;
break;
num = Message_envelope.Amqp.Decode_sequence_number();
sequence_num = sequence_num + 1;
msg_time = Message_envelope.Amqp.Decode_timestamp();
# To check timestamp of the message is following our rule
if (timestamp == 0):
if (msg_time > start_time):
print("Expected timestamp " + str(start_time) + " is not received.");
Exit_Code = -1;
break;
else:
timestamp = msg_time;
else:
if (msg_time - timestamp != span):
print("Message is not received as published sequence.")
Exit_Code = -1;
break;
else:
if msg_time == 1470239999000000:
break;
else:
timestamp = msg_time;
if (sequence_num % 100 == 0):
print("Total messages received is " + str(sequence_num))
print("Sequence number for the last record is " + str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic, id):
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = Skip_messages_from_start_size
global Running
while (True):
global Exit_Code
Message_envelope = yield Subscriber.Next()
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
if (num != sequence_num):
Exit_Code = -1
print("Unexpected sequence number. Wanted: " +
str(sequence_num) + ", got: " + str(num))
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
Running = Running - 1
except Exception, e:
print "Error: " + str(e)
def Run_subscribing_loop(topic, id, span=0):
global Running
global Exit_Code
try:
if (span == 0):
span = 50000
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
timestamp = 0
while (True):
Message_envelope = yield Subscriber.Next(30)
if Message_envelope is None:
raise RuntimeError("Subscriber.Next() timeout unexpected")
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
else:
if (msg_time - timestamp == (60 + id) * span):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
Contests = Message_envelope.Message
Contest = Contests.Get_contest_element("Some_contest_id_key")
Contest_name = Contest.Get_contestName()
element = "Some contest name " + str(sequence_num)
length = len(element)
msg_length = length
power = (id % 10) + 1
for i in range(power):
msg_length = msg_length * 2
temp = Contest_name.Get_value()
# To check if the content length is correct or not.
if (len(temp) != msg_length):
Exit_Code = -1
break
# To check content is correct or not
# From the Publisher method, we know the message should like "ElementElementElement ..."
for i in range(power):
if (temp.find(element, i * length) != i * length):
Exit_Code = -1
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic, id):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
timestamp = 0
while (True):
Message_envelope = yield Subscriber.Next()
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
else:
if (msg_time - timestamp == (600 + id) * 1000000):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
# To check messages include different types.
if (
sequence_num % 4 == 1 or sequence_num % 4 == 2
): # See publisher code above, they should have type name "Contests"
if (Message_envelope.Type_name != "Contests"):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic + "' should have type name Contests")
break
else: # See publisher code above, they should have type name "Contest"
if (Message_envelope.Type_name != "Contest"):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic + "' should have type name Contest")
break
# To check message include a not None stream_id
if (
sequence_num % 4 == 2 or sequence_num % 4 == 0
): # See publisher code above, they should have stream_id flag ORed.
if not (Message_envelope.Flags
& Message_envelope.HAS_STREAM_ID):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic + "' does not has streame_id!")
break
else: # See publisher code above, they should not have stream_id flag ORed.
if (Message_envelope.Flags & Message_envelope.HAS_STREAM_ID):
Exit_Code = -1
print("Message " + str(sequence_num) + " in topic '" +
topic +
"' includes stream_id, that is not expected.")
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
def Run_subscribing_loop(topic, id, days_ago):
global Running
global Exit_Code
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Basic_traceRoute_factory())
yield Subscriber.Open("127.0.0.1", 5672, 600)
print "[" + datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + "] Opened Synapse"
yield Subscriber.Subscribe(
topic,
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2016, 1, 1, 0, 0, 0)),
Supply_metadata=True))
sequence_num = 0
global Total_Topic
timestamp = 0
while (True):
Message_envelope = yield Subscriber.Next()
num = Message_envelope.Amqp.Decode_sequence_number()
sequence_num = sequence_num + 1
msg_time = Message_envelope.Amqp.Decode_timestamp()
# To check timestamp of the message is following our rule
if (timestamp == 0):
timestamp = msg_time
current = datetime.datetime.utcnow()
temp = Datetime_to_synapse_timestamp(current)
temp = temp - (days_ago - 1) * 24 * 60 * 60 * 1000000
if (timestamp > temp):
print("Message timestamp is not as expected " +
str(days_ago) + " ago.")
Exit_Code = -1
break
else:
if (msg_time - timestamp == (60 + (id % Total_Topic)) *
1000000):
timestamp = msg_time
else:
print("Message timestamp " + str(msg_time) +
" is not the expected value.")
Exit_Code = -1
break
if (num % 100 == 0):
print(str(id) + ": Sequence number is " + str(num))
# To check sequence number is following our rule
if (num != sequence_num):
Exit_Code = -1
break
if (sequence_num == Total_Message):
if (Exit_Code != -1):
Exit_Code = 0
break
print(
str(id) + ": Sequence number for the last record is " +
str(sequence_num))
except Exception, e:
print "Error: " + str(e)
Exit_Code = -1
except Exception, e:
print "Error: " + str(e)
print "Expected indeed..."
try:
Subscriber = Synapse_subscriber(Basic_waypoints_factory(),
Message_factory_type())
yield Subscriber.Open(Synapse_host, Synapse_port, 5)
yield Subscriber.Subscribe(
"a ",
Synapse_subscription_options(
Begin_utc=Datetime_to_synapse_timestamp(
datetime.datetime(2015, 01, 01, 0, 0)),
Supply_metadata=True))
while (True):
Message_envelope = yield Subscriber.Next()
break
except Exception, e:
print "Error: " + str(e)
print "Unexpected..."
Exit_code = -1
reactor.stop()
@defer.inlineCallbacks
def Run_publish_loop():
global Synapse_host, Synapse_port, Raise_loops_awaiting_fence, Try_quit_on_lowering_of_loops_awaiting_fence
try:
Publisher = Synapse_publisher()