Example #1
0
    def upload_segment(self, segmentID, data):
        self._getStorageNodeStatistics(len(data))
        # we can pick a storage node based on several different factors,
        # such as amount of free space, # of segments, etc

        if self._best_nodes == []:
            # we have a problem, no nodes can store data.
            print("Error, we can't store any more data in the Grid")
            return "Error, we can't store any more data in the Grid"

        _node = self._best_nodes[0]
        if len(self._best_nodes) > 1:
            _node = self._best_nodes[1:]
        else:
            self._best_nodes = []

        # put data on given node.
        _url = "https://%s/PutSegment/%s" % (_node, segmentID)
        _result = misc.access_url(_url, data=data)
        return _result
Example #2
0
   def _process_group(self, group_dict, locations):
       _main_segments=[]
       _restore_segments=[]
       for _segment_dict in group_dict['segments']:
           if _segment_dict['restoreBlock']:
              _restore_segments.append(_segment_dict['segmentID'])
           else:
              _main_segments.append(_segment_dict['segmentID'])

       _missing_segments=[]

       #now download each segment
       _count=0
       _retrieved_segments=[]
       _segments=[]
       for _segment in _main_segments:
           if locations[_segment]==[]:
              #this segment isn't located anywhere..
              _missing_segments.append(_segment)

           for _nodeID in locations[_segment]:
               _data=misc.access_url("https://%s/GetSegment/%s" % (_nodeID, _segment))

               if _data is not None:
                  if self._is_verified(_segment, _data):
                     # verified that _data == _segmentID
                     _retrieved_segments.append(_count)
                     _segments.append(_data)
                     #can ignore rest of the locations and download next segment
                     break
           _count+=1

       _num_missing_segments=len(_missing_segments)

       if _num_missing_segments==0:
          if self._decoder_padding > 0:
             #remove padding from the end of the group
             _segments[-1]=_segments[-1][:-self._decoder_padding]

          #return data since recovery isn't needed
          return True, b''.join(_segments)

       #retrieve blocks to do recovery
       _recovery_segments=[]
       while _restore_segments != []:
             _segment=_restore_segments.pop(0)

             for _nodeID in locations[_segment]:
                 _data=misc.access_url("https://%s/GetSegment/%s" % (_nodeID, _segment))

                 if _data is not None:
                    #need to verify that _data == _segmentID
                    #if so
                    _retrieved_segments.append(_count)
                    _recovery_segments.append(_data)

                 #if we have enough recovery blocks lets recover
                 if len(_retrieved_segments) >= _num_missing_segments:
                    _decoder=easyfec.Decoder(self._decoder_k, self._decoder_m)
                    return True, b''.join(_decoder.decode(_segments + _recovery_segments, 
                                           _retrieved_segments,
                                           self._decoder_padding))

       #give up, we cannot recover this segment group
       return False, 'Error! Cannot recover this group segment'