class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic,sid = struct.unpack('>bI',message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = Util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA)
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_for_topic(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_for_topic(topic, record)
self.assertMessageIsSame(message, record ,schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record ,schema_id)
class TestMessageSerializer(unittest.TestCase):
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def assertMessageIsSame(self, message, expected, schema_id):
self.assertTrue(message)
self.assertTrue(len(message) > 5)
magic, sid = struct.unpack('>bI', message[0:5])
self.assertEqual(magic, 0)
self.assertEqual(sid, schema_id)
decoded = self.ms.decode_message(message)
self.assertTrue(decoded)
self.assertEqual(decoded, expected)
def test_encode_with_schema_id(self):
adv = Util.parse_schema_from_string(data_gen.ADVANCED_SCHEMA)
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(schema_id, record)
self.assertMessageIsSame(message, record, schema_id)
subject = 'test_adv'
adv_schema_id = self.client.register(subject, adv)
self.assertNotEqual(adv_schema_id, schema_id)
records = data_gen.ADVANCED_ITEMS
for record in records:
message = self.ms.encode_record_with_schema_id(
adv_schema_id, record)
self.assertMessageIsSame(message, record, adv_schema_id)
def test_encode_record_for_topic(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_for_topic(topic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_encode_record_with_schema(self):
topic = 'test'
basic = Util.parse_schema_from_string(data_gen.BASIC_SCHEMA)
subject = 'test-value'
schema_id = self.client.register(subject, basic)
records = data_gen.BASIC_ITEMS
for record in records:
message = self.ms.encode_record_with_schema(topic, basic, record)
self.assertMessageIsSame(message, record, schema_id)
def test_select(cluster):
# type: (ClickHouseCluster) -> None
schema_registry_client = cluster.schema_registry_client
serializer = MessageSerializer(schema_registry_client)
schema = avro.schema.make_avsc_object({
'name':
'test_record',
'type':
'record',
'fields': [{
'name': 'value',
'type': 'long'
}]
})
buf = io.BytesIO()
for x in range(0, 3):
message = serializer.encode_record_with_schema('test_subject', schema,
{'value': x})
buf.write(message)
data = buf.getvalue()
instance = cluster.instances["dummy"] # type: ClickHouseInstance
schema_registry_url = "http://{}:{}".format(cluster.schema_registry_host,
cluster.schema_registry_port)
run_query(instance,
"create table avro_data(value Int64) engine = Memory()")
settings = {'format_avro_schema_registry_url': schema_registry_url}
run_query(instance, "insert into avro_data format AvroConfluent", data,
settings)
stdout = run_query(instance, "select * from avro_data")
assert list(map(str.split, stdout.splitlines())) == [
["0"],
["1"],
["2"],
]
def setup(self): ''' Initializes de hand @return: True if OK, False otherwise ''' ''' Initialize the client ''' self.client = CachedSchemaRegistryClient(url='http://localhost:8081') #self.client = CachedSchemaRegistryClient(url='http://eagle5.di.uoa.gr:8081') ''' Create Serializer ''' self.serializer = MessageSerializer(self.client) self.initialized = True return 0
class StdOutListener(tweepy.StreamListener):
def on_status(self, message):
logging.debug(datetime.now().strftime("%A, %d. %B %Y %I:%M%p"))
try:
mls = createPayload(message)
print mls
timevar = datetime.utcnow() - datetime.strptime(
mls[11], '%Y-%m-%d %H:%M:%S')
print datetime.utcnow(), datetime.strptime(mls[11],
'%Y-%m-%d %H:%M:%S')
print "Minutes and Seconds : ", divmod(
timevar.days * 86400 + timevar.seconds, 60)
except Exception, e:
logging.debug(
'There was an error in creating the payload. The error is: %s'
% e)
print 'Error in Payload Creation : ', str(e)
twitter_utils.sendErrorMail(
'There was an error in creating the payload. The error is %s' %
e)
return True
try:
#converts the payload into the avro format in preparation for loading into hbase
avro_schema = Util.parse_schema_from_string(
open('/**/**/twitter.avsc').read())
client = CachedSchemaRegistryClient(url='http://192.168.**:8081')
schema_id = client.register('twitter_avro_schema_stream4',
avro_schema)
avro_schema = client.get_by_id(schema_id)
schema_id, avro_schema, schema_version = client.get_latest_schema(
'twitter_avro_schema_stream4')
schema_version = client.get_version('twitter_avro_schema_stream4',
avro_schema)
serializer = MessageSerializer(client)
encoded = serializer.encode_record_with_schema(
topicname, avro_schema, {
"authid": mls[0],
"screen_name": mls[1],
"description": mls[2],
"favourites_count": convert_long(mls[3]),
"followers_count": convert_long(mls[4]),
"friends_count": convert_long(mls[5]),
"listed_count": convert_long(mls[6]),
"location": mls[7],
"id_str": mls[8],
"time_zone": mls[9],
"statuses_count": convert_long(mls[10]),
"created_at": mls[11],
"favorite_count": convert_long(mls[12]),
"tid": mls[13],
"in_reply_to_status_id_str": mls[14],
"in_reply_to_user_id_str": mls[15],
"lang": mls[16],
"possibly_sensitive": mls[17],
"retweet_count": convert_long(mls[18]),
"text": mls[19],
"entities_url": mls[20],
"entities_expanded_url": mls[21],
"entities_media_url": mls[22],
"disgust": convert_long(mls[23]),
"fear": convert_long(mls[24]),
"sadness": convert_long(mls[25]),
"surprise": convert_long(mls[26]),
"trust": convert_long(mls[27]),
"negative": convert_long(mls[28]),
"positive": convert_long(mls[29]),
"neutral": convert_long(mls[30]),
"celebrities": (mls[31]),
"events": (mls[32]),
"brands": (mls[33]),
"accessories": (mls[34])
})
except Exception, e:
logging.debug(
'There was an error in the generation of the avro file. The error is: %s'
% e)
print 'Error in avro generation : ', e
print mls
twitter_utils.sendErrorMail(
'There was an error in the generation of the avro file. The error is %s. This is likely due to an error in the schema. Please check the schema file under twitter_avro_schema.avsc'
% e)
return True
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
class RKConsumer:
def __init__(self, args):
self.node_name = rospy.get_name().replace('/','')
self.desired_freq = args['desired_freq']
# Checks value of freq
if self.desired_freq <= 0.0 or self.desired_freq > MAX_FREQ:
rospy.loginfo('%s::init: Desired freq (%f) is not possible. Setting desired_freq to %f'%(self.node_name,self.desired_freq, DEFAULT_FREQ))
self.desired_freq = DEFAULT_FREQ
self.real_freq = 0.0
# Saves the state of the component
self.state = State.INIT_STATE
# Saves the previous state
self.previous_state = State.INIT_STATE
# flag to control the initialization of the component
self.initialized = False
# flag to control the initialization of ROS stuff
self.ros_initialized = False
# flag to control that the control loop is running
self.running = False
# Variable used to control the loop frequency
self.time_sleep = 1.0 / self.desired_freq
# State msg to publish
self.msg_state = State()
# Timer to publish state
self.publish_state_timer = 1
self.t_publish_state = threading.Timer(self.publish_state_timer, self.publishROSstate)
self.location_x = 0.0
self.location_y = 0.0
self.goal_received = False
self.goal_sent = False
self.abort_received = False
self.rp = rospkg.RosPack()
self.tranformWGS = WGS.WGS84toNED()
def setup(self):
'''
Initializes de hand
@return: True if OK, False otherwise
'''
'''
Initialize the client
'''
self.client = CachedSchemaRegistryClient(url='http://*****:*****@return: 0 if it's performed successfully, -1 if there's any problem or the component is running
'''
if self.running or not self.initialized:
return -1
rospy.loginfo('%s::shutdown'%self.node_name)
# Cancels current timers
self.t_publish_state.cancel()
self._state_publisher.unregister()
self.initialized = False
return 0
def rosShutdown(self):
'''
Shutdows all ROS components
@return: 0 if it's performed successfully, -1 if there's any problem or the component is running
'''
if self.running or not self.ros_initialized:
return -1
# Performs ROS topics & services shutdown
self._state_publisher.unregister()
self.ros_initialized = False
return 0
def stop(self):
'''
Creates and inits ROS components
'''
self.running = False
return 0
def start(self):
'''
Runs ROS configuration and the main control loop
@return: 0 if OK
'''
self.rosSetup()
if self.running:
return 0
self.running = True
self.controlLoop()
return 0
def controlLoop(self):
'''
Main loop of the component
Manages actions by state
'''
while self.running and not rospy.is_shutdown():
t1 = time.time()
if self.state == State.INIT_STATE:
self.initState()
elif self.state == State.STANDBY_STATE:
self.standbyState()
elif self.state == State.READY_STATE:
self.readyState()
elif self.state == State.EMERGENCY_STATE:
self.emergencyState()
elif self.state == State.FAILURE_STATE:
self.failureState()
elif self.state == State.SHUTDOWN_STATE:
self.shutdownState()
self.allState()
t2 = time.time()
tdiff = (t2 - t1)
t_sleep = self.time_sleep - tdiff
if t_sleep > 0.0:
try:
rospy.sleep(t_sleep)
except rospy.exceptions.ROSInterruptException:
rospy.loginfo('%s::controlLoop: ROS interrupt exception'%self.node_name)
self.running = False
t3= time.time()
self.real_freq = 1.0/(t3 - t1)
self.running = False
# Performs component shutdown
self.shutdownState()
# Performs ROS shutdown
self.rosShutdown()
rospy.loginfo('%s::controlLoop: exit control loop'%self.node_name)
return 0
def rosPublish(self):
'''
Publish topics at standard frequency
'''
return 0
def initState(self):
'''
Actions performed in init state
'''
if not self.initialized:
self.setup()
else:
self.switchToState(State.STANDBY_STATE)
return
def standbyState(self):
'''
Actions performed in standby state
'''
self.switchToState(State.READY_STATE)
return
def readyState(self):
'''
Actions performed in ready state
'''
# decode a message from kafka
try:
for msg in self.goal_consumer:
if msg.partition == 6:
goal_decoded_object = self.serializer.decode_message(msg.value)
goal_location = goal_decoded_object.get('location')
goal_header = goal_decoded_object.get('header')
#(41.186809, -8.703597) origin parking lot/pass to radians
#(41.1872237, -8.7040693) origin corridor/pass to radians
#pointA = {'latitude' : 0.718845850137 , 'longitude' : -0.151905701075,'height' : 0}
pointA = {'latitude' : 0.718852663 , 'longitude' : -0.151914668,'height' : 0}
pointB = {}
pointB['latitude'] = goal_location.get('latitude')
pointB['longitude'] = goal_location.get('longitude')
pointB['height'] = goal_location.get('height')
ned = self.tranformWGS.displacement(pointA,pointB)
print ned
goal_x = ned.get('north')
goal_y = ned.get('east')
goal_time = goal_header.get('time')
self.goal_received = True
print ("Goal Received")
break
else:
print ("Nothing in partition 6")
break
except :
pass
'''
try:
for msg in self.abort_consumer:
abort_decoded_object = self.serializer.decode_message(msg.value)
abort_header = abort_decoded_object.get('header')
abort_time = abort_header.get('time')
self.abort_received = True
print ("Abort command Received")
break
except :
pass
'''
if self.goal_received:
#Simple Action Client
self.sac = actionlib.SimpleActionClient('move_base', MoveBaseAction )
#create goal
goal = MoveBaseGoal()
#set goal
goal.target_pose.pose.position.x = goal_x
goal.target_pose.pose.position.y = -1 * goal_y
goal.target_pose.pose.orientation.w = 1
print goal.target_pose.pose
goal.target_pose.header.frame_id = 'odom'
goal.target_pose.header.stamp = rospy.Time.now()
#start listner
print ("Waiting for server to come up")
self.sac.wait_for_server()
#send goal
self.sac.send_goal(goal)
print ("Goal Sent")
self.goal_received = False
self.goal_sent = True
#finish
if self.goal_sent:
self.sac.wait_for_result(rospy.Duration.from_sec(5.0))
#print self.sac.get_state()
if self.sac.get_state()== 3 :
rospy.loginfo("the base reached the goal")
self.goal_sent = False
print self.sac.get_result()
if (self.sac.get_state()!= 1 and self.sac.get_state()!= 3) or self.abort_received :
self.sac.cancel_goal()
rospy.loginfo("Navigation Failed")
self.goal_sent = False
self.abort_received = False
print self.sac.get_result()
return
def shutdownState(self):
'''
Actions performed in shutdown state
'''
if self.shutdown() == 0:
self.switchToState(State.INIT_STATE)
return
def emergencyState(self):
'''
Actions performed in emergency state
'''
return
def failureState(self):
'''
Actions performed in failure state
'''
return
def switchToState(self, new_state):
'''
Performs the change of state
'''
if self.state != new_state:
self.previous_state = self.state
self.state = new_state
rospy.loginfo('%s::switchToState: %s'%(self.node_name, self.stateToString(self.state)))
return
def allState(self):
'''
Actions performed in all states
'''
self.rosPublish()
return
def stateToString(self, state):
'''
@param state: state to set
@type state: State
@returns the equivalent string of the state
'''
if state == State.INIT_STATE:
return 'INIT_STATE'
elif state == State.STANDBY_STATE:
return 'STANDBY_STATE'
elif state == State.READY_STATE:
return 'READY_STATE'
elif state == State.EMERGENCY_STATE:
return 'EMERGENCY_STATE'
elif state == State.FAILURE_STATE:
return 'FAILURE_STATE'
elif state == State.SHUTDOWN_STATE:
return 'SHUTDOWN_STATE'
else:
return 'UNKNOWN_STATE'
def publishROSstate(self):
'''
Publish the State of the component at the desired frequency
'''
self.msg_state.state = self.state
self.msg_state.state_description = self.stateToString(self.state)
self.msg_state.desired_freq = self.desired_freq
self.msg_state.real_freq = self.real_freq
self._state_publisher.publish(self.msg_state)
self.t_publish_state = threading.Timer(self.publish_state_timer, self.publishROSstate)
self.t_publish_state.start()
"""
def setUp(self):
# need to set up the serializer
self.client = MockSchemaRegistryClient()
self.ms = MessageSerializer(self.client)
def get_message_serializer(self):
schema_registry_url = self.get_schema_registry_url()
logger.debug('loading schema registry: ' + schema_registry_url)
schema_client = CachedSchemaRegistryClient(url=schema_registry_url)
return MessageSerializer(schema_client)
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
from sys import argv
from config import KAFKA_URL, KAFKA_BROKER_LIST
topic = argv[1] #enter topic as parameter when running script
schema_registry_url = argv[
2] # enter schema registry url (ex. http://localhost:8081)
if len(argv) > 3 and argv[3] == 'reset':
auto_offset_reset = 'smallest'
else:
auto_offset_reset = 'largest'
schema_registry_client = CachedSchemaRegistryClient(url=schema_registry_url)
serializer = MessageSerializer(schema_registry_client)
# simple decode to replace Kafka-streaming's built-in decode decoding UTF8 ()
def decoder(s):
decoded_message = serializer.decode_message(s)
return decoded_message
# Spark Streaming from Kafka
master = 'local[2]'
app_name = 'kafka_consumer'
sc = SparkContext(master, app_name)
ssc = StreamingContext(sc, 60)
kvs = KafkaUtils.createDirectStream(ssc, [topic], {
"metadata.broker.list": KAFKA_BROKER_LIST,
def writeToavro(p, mls):
#converts the payload into the avro format in preparation for loading into hbase
try:
avro_schema = Util.parse_schema_from_string(
open('/root/quest/twitter_avro_schema.avsc').read())
client = CachedSchemaRegistryClient(url='http://192.168.111.12:8081')
schema_id = client.register('twitter_avro__schema_stream4',
avro_schema)
avro_schema = client.get_by_id(schema_id)
schema_id, avro_schema, schema_version = client.get_latest_schema(
'twitter_avro__schema_stream4')
schema_version = client.get_version('twitter_avro__schema_stream4',
avro_schema)
serializer = MessageSerializer(client)
encoded = serializer.encode_record_with_schema(
topicname, avro_schema, {
"authid": mls[0],
"screen_name": mls[1],
"description": mls[2],
"favourites_count": convert_long(mls[3]),
"followers_count": convert_long(mls[4]),
"friends_count": convert_long(mls[5]),
"listed_count": convert_long(mls[6]),
"location": mls[7],
"id_str": mls[8],
"time_zone": mls[9],
"statuses_count": convert_long(mls[10]),
"created_at": mls[11],
"favorite_count": convert_long(mls[12]),
"tid": mls[13],
"in_reply_to_status_id_str": mls[14],
"in_reply_to_user_id_str": mls[15],
"lang": mls[16],
"possibly_sensitive": mls[17],
"retweet_count": convert_long(mls[18]),
"text": mls[19],
"entities_url": mls[20],
"entities_expanded_url": mls[21],
"entities_media_url": mls[22],
"disgust": convert_long(mls[23]),
"fear": convert_long(mls[24]),
"sadness": convert_long(mls[25]),
"surprise": convert_long(mls[26]),
"trust": convert_long(mls[27]),
"negative": convert_long(mls[28]),
"positive": convert_long(mls[29]),
"neutral": convert_long(mls[30]),
"celebrities": (mls[31]),
"events": (mls[32]),
"brands": (mls[33]),
"accessories": (mls[34])
})
except Exception, e:
logging.debug(
'There was an error in the generation of the avro file. The error is: %s'
% e)
print 'Error in avro generation : ', e
print mls
twitter_utils.sendErrorMail(
'There was an error in the generation of the avro file. The error is %s'
% e)
return True
