def process(self):
# fix this, each group should be in its own list for easier processing
# when we reassemble the file
_fp = open(self._filename, "rb")
_group_count = 0
_metadata = []
while True:
_data = _fp.read(self._group_size)
if not _data:
break
_parms = ZfecParameterCalculator(_group_count)
_k, _m = _parms.get_parameters()
_encoder = easyfec.Encoder(_k, _m)
_segments = _encoder.encode(_data)
_segment_length = len(_segments[0])
# padding should be 0 if _k evenly divides into group size
_padding = _segment_length * _k - len(_data)
_recovery_data = {"type": "zfec", "group": _group_count, "k": _k, "m": _m, "padding": _padding}
_seq = 0
_blocks = []
for _segment in _segments:
_segmentID = misc.get_shaID(_segment)
# _sha=hashlib.new('sha1')
# _sha.update(_segment)
# _segmentID=_sha.hexdigest()
_restoreBlock = False
if _seq >= _k:
_restoreBlock = True
_blocks.append(
{
"group": _group_count,
"seq": _seq,
"segmentID": _segmentID,
"length": len(_segment),
"restoreBlock": _restoreBlock,
}
)
self._upload_file.upload_segment(_segmentID, _segment)
_seq += 1
print(_group_count, _seq, _segmentID)
_metadata.append({"segments": _blocks, "recovery": _recovery_data})
_group_count += 1
# write out segment to temp storage so we can upload the data
# and metadata. (or just upload segment to one or more
# storage nodes)
_fp.close()
_json = json.dumps(_metadata).encode("utf-8")
# _shaID=misc.get_shaID(_json)
_result = self._mdn_handler.send_message(
"/Client/putfile/%s/%s" % (self._parentID, self._grid_filename), data=_json
)
# _url="https://%s/Client/putfile/%s/%s" % (self._metadataNode,
# self._parentID, self._grid_filename)
# _result=misc.access_url(_url, data=_json)
return _result
def MetadataNode(environ, start_response):
_path_info=environ["PATH_INFO"]
print(_path_info)
if _path_info.startswith('/StorageNode/Register/'):
#this is a storage node register
# syntax /Register/ipaddr:port/
#print(_segmentLocator.get_statistics())
_ipaddr=_path_info[22:]
#print(_ipaddr)
_segmentLocator.refresh(_ipaddr)
start_response('200 OK', [('Content-type', 'text/plain')])
return [''.encode('utf-8')]
#elif _path_info.startswith('/RegisterMetadataNode/'):
# #this is a storage node register
# # syntax /Register/ipaddr:port/
# _ipaddr=_path_info[22:]
# self._metadataNodes.refresh(_ipaddr)
elif _path_info.startswith('/Monitor'):
#return a nice looking page with basic info such as
#the number of storage nodes, the number of segments
#each is storing, the last time they checked in, etc
_fp=open("templates/default.tmpl", "r")
_nodes_dict=_segmentLocator.get_statistics()
_storagenodes=[]
_mdn=[]
_missing_segments=[]
for _node in _nodes_dict.keys():
_data=_nodes_dict[_node]
_data['timedelta']=misc.convert_delta_time(time.time() - _data['timestamp'])
_data['timestamp']=misc.convert_seconds(_data['timestamp'])
_data['free']=misc.convert_bytes(_data['free'])
_data['used']=misc.convert_bytes(_data['used'])
_storagenodes.append(_nodes_dict[_node])
_data={'last_updated': datetime.datetime.now(),
'storagenodes': _storagenodes,
'metadatanodes': _mdn,
'missingsegments': _fsmetadata.get_statistics()}
_out=io.StringIO()
_tmpl=XYAPTU.xcopier(_data, ouf=_out)
_tmpl.xcopy(_fp)
start_response('200 OK', [('Content-type', 'text/html')])
return [_out.getvalue().encode('utf-8')]
elif _path_info.startswith('/Client'):
#this is from a client that will want certain information
#command list (listdir, rmdir, mkdir)
# (put file, get file)
_cmd=_path_info[7:]
if _cmd.startswith('/locate_segments'):
_result=misc.get_wsgi_file_contents(environ)
_segments=misc.receive_compressed_response(_result)
#_segments=json.loads(_result.decode('utf-8'))
_dict=_segmentLocator.locate_segments(_segments)
start_response('200 OK', [('Content-type', 'text/plain')])
return [misc.send_compressed_response(_dict)]
elif _cmd.startswith('/listdir/'):
_id=_cmd[9:]
#print(_id)
_result=_fsmetadata.listdir(_id)
if _result is None:
start_response('404 OK', [('Content-type', 'text/plain')])
else:
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/Name2ID/'):
_name=_cmd[9:]
#print(_name)
_result=_fsmetadata.lookupID(_name)
print(_result)
if _result is None:
start_response('404 OK', [('Content-type', 'text/plain')])
else:
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/rmdir/'):
_parentID, _id, _name=_cmd[7:].split('/',2)
_result=_fsmetadata.rmdir(_parentID, _id, _name)
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/cp/'):
#copy metadata within the grid (ie, from grid location to grid location)
_parentID, _shaID, _target_dir_id, _name=_cmd[4:].split('/',3)
_result=_fsmetadata.cp(_parentID, _shaID, _target_dir_id, _name)
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/mv/'):
#copy metadata within grid
_parentID, _shaID, _target_dir_id, _name=_cmd[4:].split('/',3)
_result=_fsmetadata.cp(_parentID, _shaID, _target_dir_id, _name)
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/mkdir/'):
_parentid, _name_id, _name=_cmd[7:].split('/', 2)
print(_parentid, _name_id, _name)
_result=_fsmetadata.mkdir(_parentid, _name_id, _name)
if _result is None:
start_response('200 OK', [('Content-type', 'text/plain')])
_result=''
else:
print(_result)
start_response('404 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/putfile/'):
_parentID, _file_name=_cmd[9:].split('/',1)
_file_metadata=misc.get_wsgi_file_contents(environ)
#get file metadata
if _file_metadata is not None:
_shaID=misc.get_shaID(_file_metadata)
_size=0
_result=_fsmetadata.putfile(_parentID, _shaID, _file_name, _size)
#print(_result)
if _result is None:
#_metadata=json.loads(_file_metadata.decode('utf-8'))
_fsmetadata.add_file_metadata(_file_metadata)
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
print(_result)
_result=''
start_response('404 OK', [('Content-type', 'text/plain')])
return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/getfile/'):
_shaID=_cmd[9:]
_valid, _result=_fsmetadata.get_file_metadata(_shaID)
if _result is None:
start_response('404 OK', [('Content-type', 'text/plain')])
_result=''
else:
start_response('200 OK', [('Content-type', 'text/plain')])
return [misc.send_compressed_response(_result.decode('utf-8'))]
#return [json.dumps(_result).encode('utf-8')]
elif _cmd.startswith('/getStorageNodeStatistics'):
_dict=_segmentLocator.get_statistics()
if _dict=={}: #try refreshing to get a list of nodes
_segmentLocator.refresh()
_dict=_segmentLocator.get_statistics()
#send string to client
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps(_dict).encode('utf-8')]
elif _cmd.startswith('/rmfile/'):
_dir_id, _shaID=_cmd[8:].split('/')
_result=_fsmetadata.rmfile(_dir_id, _shaID)
#if _fsmetadata.get_file_locations(_shaID)==[]:
#this file isn't in the metadata anymore so lets
#remove the segments
#_fsmetadata.rmfile(_shaID)
if _result is None:
start_response('200 OK', [('Content-type', 'text/plain')])
return [json.dumps('').encode('utf-8')]
else:
start_response('404 OK', [('Content-type', 'text/plain')])
return [json.dumps(_dict).encode('utf-8')]
start_response('404 OK', [('Content-type', 'text/plain')])
return [json.dumps('').encode('utf-8')]