def upload_segment(self, segmentID, data):
self._getStorageNodeStatistics(len(data))
# we can pick a storage node based on several different factors,
# such as amount of free space, # of segments, etc
if self._best_nodes == []:
# we have a problem, no nodes can store data.
print("Error, we can't store any more data in the Grid")
return "Error, we can't store any more data in the Grid"
_node = self._best_nodes[0]
if len(self._best_nodes) > 1:
_node = self._best_nodes[1:]
else:
self._best_nodes = []
# put data on given node.
_url = "https://%s/PutSegment/%s" % (_node, segmentID)
_result = misc.access_url(_url, data=data)
return _result
def _process_group(self, group_dict, locations):
_main_segments=[]
_restore_segments=[]
for _segment_dict in group_dict['segments']:
if _segment_dict['restoreBlock']:
_restore_segments.append(_segment_dict['segmentID'])
else:
_main_segments.append(_segment_dict['segmentID'])
_missing_segments=[]
#now download each segment
_count=0
_retrieved_segments=[]
_segments=[]
for _segment in _main_segments:
if locations[_segment]==[]:
#this segment isn't located anywhere..
_missing_segments.append(_segment)
for _nodeID in locations[_segment]:
_data=misc.access_url("https://%s/GetSegment/%s" % (_nodeID, _segment))
if _data is not None:
if self._is_verified(_segment, _data):
# verified that _data == _segmentID
_retrieved_segments.append(_count)
_segments.append(_data)
#can ignore rest of the locations and download next segment
break
_count+=1
_num_missing_segments=len(_missing_segments)
if _num_missing_segments==0:
if self._decoder_padding > 0:
#remove padding from the end of the group
_segments[-1]=_segments[-1][:-self._decoder_padding]
#return data since recovery isn't needed
return True, b''.join(_segments)
#retrieve blocks to do recovery
_recovery_segments=[]
while _restore_segments != []:
_segment=_restore_segments.pop(0)
for _nodeID in locations[_segment]:
_data=misc.access_url("https://%s/GetSegment/%s" % (_nodeID, _segment))
if _data is not None:
#need to verify that _data == _segmentID
#if so
_retrieved_segments.append(_count)
_recovery_segments.append(_data)
#if we have enough recovery blocks lets recover
if len(_retrieved_segments) >= _num_missing_segments:
_decoder=easyfec.Decoder(self._decoder_k, self._decoder_m)
return True, b''.join(_decoder.decode(_segments + _recovery_segments,
_retrieved_segments,
self._decoder_padding))
#give up, we cannot recover this segment group
return False, 'Error! Cannot recover this group segment'
