def Run_subscribing_loop(topic, total_messages): global Running global Exit_Code try: Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open(Synapse_host, Synapse_port) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" yield Subscriber.Subscribe( topic, Synapse_subscription_options(Begin_utc=1, Supply_metadata=True)) count = 0 while (True): Message_envelope = yield Subscriber.Next(30) if Message_envelope is None: raise RuntimeError("Subscriber.Next() timeout unexpected") count += 1 if (count % 1000 == 0): print("Received " + str(count) + " messages, topic=" + topic) if (count >= total_messages): print("All messages are received for topic: " + topic) break except Exception, e: print "Error: " + str(e) Exit_Code = -1
def Run_subscribing_loop(topic, id): global Running global Exit_Code try: Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open("127.0.0.1", 5672) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" yield Subscriber.Subscribe( topic, Synapse_subscription_options( Begin_utc=Datetime_to_synapse_timestamp( datetime.datetime(2016, 1, 1, 0, 0, 0)), Supply_metadata=True)) sequence_num = 0 global Running timestamp = 0 while (True): global Exit_Code Message_envelope = yield Subscriber.Next(15) if (not Message_envelope): if ( Exit_Code != -1 ): # Got None before receiving all messages, it is an error. Exit_Code = -1 break num = Message_envelope.Amqp.Decode_sequence_number() sequence_num = sequence_num + 1 msg_time = Message_envelope.Amqp.Decode_timestamp() # To check timestamp of the message is following our rule if (timestamp == 0): timestamp = msg_time else: if (msg_time - timestamp == (60 * 60) * 1000000): timestamp = msg_time else: print("Message timestamp " + str(msg_time) + " is not the expected value.") Exit_Code = -1 break if (num % 100 == 0): print(str(id) + ": Sequence number is " + str(num)) # To check sequence number is following our rule if (num != sequence_num): Exit_Code = -1 break if (sequence_num == Total_Message): if (Exit_Code != -1): Exit_Code = 0 break print( str(id) + ": Sequence number for the last record is " + str(sequence_num)) except Exception, e: print "Error: " + str(e) Exit_Code = -1
def Run_extend_ack_loop(topic, fullmode=True): global Running global Exit_Code global Extend_ack_in_progress try: sub = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield sub.Open(Synapse_host, Synapse_port) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" version = sub.Client.server_properties['version'] info = sub.Client.server_properties['information'] print("Synapse information:'" + info + "', version:'" + version + "'") props = {} props["Topics"] = 1 if (FullMode): print("Subscribe to full mode as well as extend_ack mode") yield sub.Subscribe( topic, Synapse_subscription_options(Begin_utc=1, Supply_metadata=True)) else: print("Subscribe to extend_ack mode only") # in this test, we just keep querying the topic related information from server yield sub.Subscribe_extended_ack(props) count = 0 fmcount = 0 while True: msg = yield sub.Next(30) if (msg is None): raise RuntimeError("Could not query topics") break if (msg.Is_ack): if (msg.Amqp.Properties.headers.has_key("Stats")): array = msg.Amqp.Properties.headers['Stats'] if (count % 1000 == 0): print("Stats:" + str(array)) if (msg.Amqp.Properties.headers.has_key("Topics")): topics = msg.Amqp.Properties.headers["Topics"] if (count % 1000 == 0): print("Topics:" + str(topics)) count += 1 # in this test, we just keep querying the topic related information from server (Not yield here) if not Extend_ack_in_progress: if (not FullMode): yield (Timeout_wrapper(60)(Subscribe_extend_ack))( sub, props) else: (Timeout_wrapper(60)(Subscribe_extend_ack))(sub, props) else: print("Extend_ack_in_progress ...") else: fmcount += 1 if (fmcount % 10000 == 0): print("Received " + str(fmcount) + " messages(non-ack) in extend_ack subscription") except Exception, e: print "Error: " + str(e) Exit_Code = -1
def SubscribeMsgs(timestamp): global Exit_Code; global Running; try : Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open(Synapse_host, Synapse_port) print "[" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "] Opened Synapse" start_time = 1470120000000000; if (timestamp != 0): start_time = timestamp * 1000000; yield Subscriber.Subscribe("test.topic", Synapse_subscription_options(Begin_utc = start_time, Supply_metadata = True)) sequence_num = 0 timestamp = 0 span = 1000000; while (True) : global Exit_Code Message_envelope = yield Subscriber.Next(30) if (Message_envelope == None): print("Waiting message timeout."); Exit_Code = -1; break; num = Message_envelope.Amqp.Decode_sequence_number(); sequence_num = sequence_num + 1; msg_time = Message_envelope.Amqp.Decode_timestamp(); # To check timestamp of the message is following our rule if (timestamp == 0): if (msg_time > start_time): print("Expected timestamp " + str(start_time) + " is not received."); Exit_Code = -1; break; else: timestamp = msg_time; else: if (msg_time - timestamp != span): print("Message is not received as published sequence.") Exit_Code = -1; break; else: if msg_time == 1470239999000000: break; else: timestamp = msg_time; if (sequence_num % 100 == 0): print("Total messages received is " + str(sequence_num)) print("Sequence number for the last record is " + str(sequence_num)) except Exception, e: print "Error: " + str(e) Exit_Code = -1
def Run_subscribing_loop(topic, id): try: Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open("127.0.0.1", 5672) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" yield Subscriber.Subscribe( topic, Synapse_subscription_options( Begin_utc=Datetime_to_synapse_timestamp( datetime.datetime(2016, 1, 1, 0, 0, 0)), Supply_metadata=True)) sequence_num = Skip_messages_from_start_size global Running while (True): global Exit_Code Message_envelope = yield Subscriber.Next() num = Message_envelope.Amqp.Decode_sequence_number() sequence_num = sequence_num + 1 if (num % 100 == 0): print(str(id) + ": Sequence number is " + str(num)) if (num != sequence_num): Exit_Code = -1 print("Unexpected sequence number. Wanted: " + str(sequence_num) + ", got: " + str(num)) break if (sequence_num == Total_Message): if (Exit_Code != -1): Exit_Code = 0 break print( str(id) + ": Sequence number for the last record is " + str(sequence_num)) Running = Running - 1 except Exception, e: print "Error: " + str(e)
def Run_subscribing_loop(topic, id, span=0): global Running global Exit_Code try: if (span == 0): span = 50000 Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open("127.0.0.1", 5672) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" yield Subscriber.Subscribe( topic, Synapse_subscription_options( Begin_utc=Datetime_to_synapse_timestamp( datetime.datetime(2016, 1, 1, 0, 0, 0)), Supply_metadata=True)) sequence_num = 0 timestamp = 0 while (True): Message_envelope = yield Subscriber.Next(30) if Message_envelope is None: raise RuntimeError("Subscriber.Next() timeout unexpected") num = Message_envelope.Amqp.Decode_sequence_number() sequence_num = sequence_num + 1 msg_time = Message_envelope.Amqp.Decode_timestamp() # To check timestamp of the message is following our rule if (timestamp == 0): timestamp = msg_time else: if (msg_time - timestamp == (60 + id) * span): timestamp = msg_time else: print("Message timestamp " + str(msg_time) + " is not the expected value.") Exit_Code = -1 break if (num % 100 == 0): print(str(id) + ": Sequence number is " + str(num)) # To check sequence number is following our rule if (num != sequence_num): Exit_Code = -1 break if (sequence_num == Total_Message): if (Exit_Code != -1): Exit_Code = 0 break Contests = Message_envelope.Message Contest = Contests.Get_contest_element("Some_contest_id_key") Contest_name = Contest.Get_contestName() element = "Some contest name " + str(sequence_num) length = len(element) msg_length = length power = (id % 10) + 1 for i in range(power): msg_length = msg_length * 2 temp = Contest_name.Get_value() # To check if the content length is correct or not. if (len(temp) != msg_length): Exit_Code = -1 break # To check content is correct or not # From the Publisher method, we know the message should like "ElementElementElement ..." for i in range(power): if (temp.find(element, i * length) != i * length): Exit_Code = -1 break print( str(id) + ": Sequence number for the last record is " + str(sequence_num)) except Exception, e: print "Error: " + str(e) Exit_Code = -1
def Run_subscribing_loop(topic, id): global Running global Exit_Code try: Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open("127.0.0.1", 5672) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" yield Subscriber.Subscribe( topic, Synapse_subscription_options( Begin_utc=Datetime_to_synapse_timestamp( datetime.datetime(2016, 1, 1, 0, 0, 0)), Supply_metadata=True)) sequence_num = 0 timestamp = 0 while (True): Message_envelope = yield Subscriber.Next() num = Message_envelope.Amqp.Decode_sequence_number() sequence_num = sequence_num + 1 msg_time = Message_envelope.Amqp.Decode_timestamp() # To check timestamp of the message is following our rule if (timestamp == 0): timestamp = msg_time else: if (msg_time - timestamp == (600 + id) * 1000000): timestamp = msg_time else: print("Message timestamp " + str(msg_time) + " is not the expected value.") Exit_Code = -1 break if (num % 100 == 0): print(str(id) + ": Sequence number is " + str(num)) # To check sequence number is following our rule if (num != sequence_num): Exit_Code = -1 break if (sequence_num == Total_Message): if (Exit_Code != -1): Exit_Code = 0 break # To check messages include different types. if ( sequence_num % 4 == 1 or sequence_num % 4 == 2 ): # See publisher code above, they should have type name "Contests" if (Message_envelope.Type_name != "Contests"): Exit_Code = -1 print("Message " + str(sequence_num) + " in topic '" + topic + "' should have type name Contests") break else: # See publisher code above, they should have type name "Contest" if (Message_envelope.Type_name != "Contest"): Exit_Code = -1 print("Message " + str(sequence_num) + " in topic '" + topic + "' should have type name Contest") break # To check message include a not None stream_id if ( sequence_num % 4 == 2 or sequence_num % 4 == 0 ): # See publisher code above, they should have stream_id flag ORed. if not (Message_envelope.Flags & Message_envelope.HAS_STREAM_ID): Exit_Code = -1 print("Message " + str(sequence_num) + " in topic '" + topic + "' does not has streame_id!") break else: # See publisher code above, they should not have stream_id flag ORed. if (Message_envelope.Flags & Message_envelope.HAS_STREAM_ID): Exit_Code = -1 print("Message " + str(sequence_num) + " in topic '" + topic + "' includes stream_id, that is not expected.") break print( str(id) + ": Sequence number for the last record is " + str(sequence_num)) except Exception, e: print "Error: " + str(e) Exit_Code = -1
def Run_subscribing_loop(topic, id, days_ago): global Running global Exit_Code try: Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Basic_traceRoute_factory()) yield Subscriber.Open("127.0.0.1", 5672, 600) print "[" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") + "] Opened Synapse" yield Subscriber.Subscribe( topic, Synapse_subscription_options( Begin_utc=Datetime_to_synapse_timestamp( datetime.datetime(2016, 1, 1, 0, 0, 0)), Supply_metadata=True)) sequence_num = 0 global Total_Topic timestamp = 0 while (True): Message_envelope = yield Subscriber.Next() num = Message_envelope.Amqp.Decode_sequence_number() sequence_num = sequence_num + 1 msg_time = Message_envelope.Amqp.Decode_timestamp() # To check timestamp of the message is following our rule if (timestamp == 0): timestamp = msg_time current = datetime.datetime.utcnow() temp = Datetime_to_synapse_timestamp(current) temp = temp - (days_ago - 1) * 24 * 60 * 60 * 1000000 if (timestamp > temp): print("Message timestamp is not as expected " + str(days_ago) + " ago.") Exit_Code = -1 break else: if (msg_time - timestamp == (60 + (id % Total_Topic)) * 1000000): timestamp = msg_time else: print("Message timestamp " + str(msg_time) + " is not the expected value.") Exit_Code = -1 break if (num % 100 == 0): print(str(id) + ": Sequence number is " + str(num)) # To check sequence number is following our rule if (num != sequence_num): Exit_Code = -1 break if (sequence_num == Total_Message): if (Exit_Code != -1): Exit_Code = 0 break print( str(id) + ": Sequence number for the last record is " + str(sequence_num)) except Exception, e: print "Error: " + str(e) Exit_Code = -1
except Exception, e: print "Error: " + str(e) print "Expected indeed..." try: Subscriber = Synapse_subscriber(Basic_waypoints_factory(), Message_factory_type()) yield Subscriber.Open(Synapse_host, Synapse_port, 5) yield Subscriber.Subscribe( "a ", Synapse_subscription_options( Begin_utc=Datetime_to_synapse_timestamp( datetime.datetime(2015, 01, 01, 0, 0)), Supply_metadata=True)) while (True): Message_envelope = yield Subscriber.Next() break except Exception, e: print "Error: " + str(e) print "Unexpected..." Exit_code = -1 reactor.stop() @defer.inlineCallbacks def Run_publish_loop(): global Synapse_host, Synapse_port, Raise_loops_awaiting_fence, Try_quit_on_lowering_of_loops_awaiting_fence try: Publisher = Synapse_publisher()