def preprocess(self, fpath):
"""read file and chunkify it to be small batch for grpc transport"""
timestamp_utc = '2017-08-08'
cnt = 10
buffer = []
print(fpath)
with open(fpath) as f:
for line in f:
print line
if not cnt:
break
if len(buffer) == CONST_CHUNK_SIZE:
print "Chunk size"
res = ''.join(buffer)
buffer = []
print res
response=Response(
Code=1,
metaData=MetaData(uuid='14829'),
datFragment=DatFragment(timestamp_utc=timestamp_utc, data=res.encode())
)
yield response
else:
buffer.append(line)
cnt = cnt - 1
def preprocess(fpath):
"""read file and chunkify it to be small batch for grpc transport"""
#timestamp_utc = '2017-08-08 12:00:00'
cnt = 10
buffer = []
with open(fpath) as f:
for line in f:
print line
if not cnt:
break
if len(buffer) == CONST_CHUNK_SIZE:
print "Chunk size"
res = ''.join(buffer)
buffer = []
print "Res is"
print res
putRequest=PutRequest(
metaData=MetaData(uuid='14829', mediaType=CONST_MEDIA_TYPE_TEXT),
datFragment=DatFragment(data=res.encode()))
request = Request(
fromSender='some put sender',
toReceiver='some put receiver',
putRequest=putRequest
)
yield res
else:
buffer.append(line)
cnt = cnt - 1
if buffer:
yield ''.join(buffer)
def put_iterator(fpath):
buffer = process_file(fpath)
for raw in buffer:
putRequest = PutRequest(metaData=MetaData(
uuid='14829', mediaType=CONST_MEDIA_TYPE_TEXT),
datFragment=DatFragment(data=raw.encode()))
request = Request(fromSender='some put sender',
toReceiver='some put receiver',
putRequest=putRequest)
yield request
def put_req_iterator(fpath, sender, receiver):
my_uuid = str(uuid.uuid1())
for raw in preprocess(fpath):
print 'raw data'
print raw
yield Request(fromSender=sender,
toReceiver=receiver,
putRequest=PutRequest(
metaData=MetaData(
uuid=my_uuid,
mediaType=CONST_MEDIA_TYPE_TEXT_MESOWEST),
datFragment=DatFragment(data=raw.encode())))
def get(self):
req = Request(fromSender='some put sender',
toReceiver='some put receiver',
getRequest=GetRequest(metaData=MetaData(uuid='14829'),
queryParams=QueryParams(
from_utc='2018-03-16 21:00:00',
to_utc='2018-03-16 23:00:00')))
resp = self.stub.getHandler(req)
print 'Response from server'
for response in resp:
print response
print response.datFragment.data
def put_iterator(fpath, timestamp=''):
buffer = process_file(fpath)
for raw in buffer:
if timestamp != '':
data_out = normalize_data_mesonet(raw, timestamp)
print data_out
else:
data_out = normalize_data_mesowest(raw)
putRequest = PutRequest(
metaData=MetaData(uuid='14829', mediaType=CONST_MEDIA_TYPE_TEXT),
datFragment=DatFragment(data=data_out.encode(),
timestamp_utc=timestamp))
request = Request(fromSender=CLIENT_IP,
toReceiver='host IP',
putRequest=putRequest)
yield request
def get(self, fp, from_utc, to_utc):
"""
Returns: bool
"""
req = Request(fromSender=self.sender,
toReceiver=self.receiver,
getRequest=GetRequest(metaData=MetaData(uuid='14829'),
queryParams=QueryParams(
from_utc=from_utc,
to_utc=to_utc)))
for resp in self.stub.getHandler(req):
if resp.code == 2:
print('read failed at this node!')
return False
else:
fp.write(resp.datFragment.data.decode())
return True
def getHandler(self, request, context):
print "Receive get request"
print request
# Get info from local mongodb
getRequest = request.getRequest
queryParams = getRequest.queryParams
from_utc = queryParams.from_utc
to_utc = queryParams.to_utc
#param_json = queryParams.param_json
# file_name = 'query' + '.txt'
# with open(file_name, 'w') as file:
# file.write(str(from_utc) + " to " + str(to_utc))
# call(['../ProjectCluster/client.sh', ' 1 -write -' + file_name])
#from_utc_format = from_utc.replace('-','').replace(' ','/').replace(':','')[:-2]
#to_utc_format = to_utc.replace('-','').replace(' ','/').replace(':','')[:-2]
print "Getting stuff from mongo db"
client = MongoClient('localhost', 27017)
db = client.pymongo_test
#insert_bulk_mongo(db, buffer)
result = find_mongo(db, from_utc, to_utc)
print "Result return from mongodb"
print result
#for document in result:
# print "TTT: Result from mongo"
# print document
for line in result:
print "Line in mongodb"
print line
response=Response(
Code=1,
metaData=MetaData(uuid='14829'),
datFragment=DatFragment(data=str(line).encode())
)
yield response