def ping(self, message):
req = Request(
fromSender="sender",
toReceiver="receiver",
ping=PingRequest(msg=message))
resp = self.stub.ping(req)
print(resp.msg)
def preprocess(fpath):
"""read file and chunkify it to be small batch for grpc transport"""
#timestamp_utc = '2017-08-08 12:00:00'
cnt = 10
buffer = []
with open(fpath) as f:
for line in f:
print line
if not cnt:
break
if len(buffer) == CONST_CHUNK_SIZE:
print "Chunk size"
res = ''.join(buffer)
buffer = []
print "Res is"
print res
putRequest=PutRequest(
metaData=MetaData(uuid='14829', mediaType=CONST_MEDIA_TYPE_TEXT),
datFragment=DatFragment(data=res.encode()))
request = Request(
fromSender='some put sender',
toReceiver='some put receiver',
putRequest=putRequest
)
yield res
else:
buffer.append(line)
cnt = cnt - 1
if buffer:
yield ''.join(buffer)
def ping(self, data):
req = Request(
fromSender=CLIENT_IP,
toReceiver=self.receiver_ip,
ping=PingRequest(msg='this is a sample ping request'))
resp = self.stub.ping(req)
return resp.msg
def stream_putreq(self, recordlist):
id = str(uuid.uuid1())
yield Request(
fromSender="sender",
toReceiver="receiver",
putRequest=PutRequest(
metaData=MetaData(uuid=id, mediaType=1),
datFragment=DatFragment(data=recordlist.encode())))
def get(self, from_time, to_time):
req = Request(
fromSender="sender",
toReceiver="receiver",
getRequest=GetRequest(
metaData=MetaData(uuid='12345'),
queryParams=QueryParams(from_utc=from_time, to_utc=to_time)))
for resp in self.stub.getHandler(req):
print(resp)
def put_req_iterator(fpath, sender, receiver):
my_uuid = str(uuid.uuid1())
for raw in parse_file(fpath):
yield Request(
fromSender=sender,
toReceiver=receiver,
putRequest=PutRequest(
metaData=MetaData(uuid=my_uuid),
datFragment=DatFragment(data=raw.encode()))
)
def ping(self, msg):
"""
Returns: bool
"""
req = Request(
fromSender=self.sender,
toReceiver=self.receiver,
ping=PingRequest(msg=msg))
resp = self.stub.ping(req)
print(resp.msg)
return True
def put_req_iterator(fpath, sender, receiver):
my_uuid = str(uuid.uuid1())
for raw in preprocess(fpath):
print 'raw data'
print raw
yield Request(
fromSender=sender,
toReceiver=receiver,
putRequest=PutRequest(
metaData=MetaData(uuid=my_uuid, mediaType=CONST_MEDIA_TYPE_TEXT_MESOWEST),
datFragment=DatFragment(data=raw.encode()))
)
def get(self):
req = Request(fromSender=CLIENT_IP,
toReceiver=self.receiver_ip,
getRequest=GetRequest(metaData=MetaData(uuid='14829'),
queryParams=QueryParams(
from_utc='2018-03-16 21:00:00',
to_utc='2018-03-16 23:00:00')))
resp = self.stub.getHandler(req)
print 'Response from server'
for response in resp:
print response
print response.datFragment.data
def put_iterator(fpath, timestamp=''):
buffer = process_file(fpath)
for raw in buffer:
if timestamp != '':
data_out = normalize_data_mesonet(raw, timestamp)
print data_out
else:
data_out = normalize_data_mesowest(raw)
putRequest = PutRequest(
metaData=MetaData(uuid='14829', mediaType=CONST_MEDIA_TYPE_TEXT),
datFragment=DatFragment(data=data_out.encode(),
timestamp_utc=timestamp))
request = Request(fromSender=CLIENT_IP,
toReceiver='host IP',
putRequest=putRequest)
yield request
def get(self, fp, from_utc, to_utc):
"""
Returns: bool
"""
req = Request(
fromSender=self.sender,
toReceiver=self.receiver,
getRequest=GetRequest(
metaData=MetaData(uuid='14829'),
queryParams=QueryParams(from_utc=from_utc,to_utc=to_utc))
)
for resp in self.stub.getHandler(req):
if resp.code == 2:
print('read failed at this node!')
return False
else:
fp.write(resp.datFragment.data.decode())
return True
def get(self, fp, from_utc, to_utc, params_json):
"""
Returns: bool
"""
req = Request(
fromSender=self.sender,
toReceiver=self.receiver,
getRequest=GetRequest(
metaData=MetaData(),
queryParams=QueryParams(from_utc=from_utc,to_utc=to_utc, params_json=params_json))
)
fp.write(CONST_MESOWEST_HEADER + CONST_NEWLINE_CHAR)
for resp in self.stub.getHandler(req):
if resp.code == 2:
print('read failed at this node!')
return False
else:
fp.write(resp.datFragment.data.decode() + CONST_NEWLINE_CHAR)
return True