Ejemplo n.º 1
0
    def process(self):
        # fix this, each group should be in its own list for easier processing
        # when we reassemble the file
        _fp = open(self._filename, "rb")
        _group_count = 0

        _metadata = []
        while True:
            _data = _fp.read(self._group_size)
            if not _data:
                break

            _parms = ZfecParameterCalculator(_group_count)
            _k, _m = _parms.get_parameters()
            _encoder = easyfec.Encoder(_k, _m)
            _segments = _encoder.encode(_data)

            _segment_length = len(_segments[0])
            # padding should be 0 if _k evenly divides into group size
            _padding = _segment_length * _k - len(_data)

            _recovery_data = {"type": "zfec", "group": _group_count, "k": _k, "m": _m, "padding": _padding}

            _seq = 0
            _blocks = []
            for _segment in _segments:
                _segmentID = misc.get_shaID(_segment)
                # _sha=hashlib.new('sha1')
                # _sha.update(_segment)
                # _segmentID=_sha.hexdigest()
                _restoreBlock = False
                if _seq >= _k:
                    _restoreBlock = True
                _blocks.append(
                    {
                        "group": _group_count,
                        "seq": _seq,
                        "segmentID": _segmentID,
                        "length": len(_segment),
                        "restoreBlock": _restoreBlock,
                    }
                )

                self._upload_file.upload_segment(_segmentID, _segment)

                _seq += 1
                print(_group_count, _seq, _segmentID)

            _metadata.append({"segments": _blocks, "recovery": _recovery_data})
            _group_count += 1

            # write out segment to temp storage so we can upload the data
            # and metadata. (or just upload segment to one or more
            # storage nodes)

        _fp.close()

        _json = json.dumps(_metadata).encode("utf-8")

        # _shaID=misc.get_shaID(_json)

        _result = self._mdn_handler.send_message(
            "/Client/putfile/%s/%s" % (self._parentID, self._grid_filename), data=_json
        )
        # _url="https://%s/Client/putfile/%s/%s" % (self._metadataNode,
        #          self._parentID, self._grid_filename)
        # _result=misc.access_url(_url, data=_json)
        return _result
Ejemplo n.º 2
0
def MetadataNode(environ, start_response):
    _path_info=environ["PATH_INFO"]
    print(_path_info)
    if _path_info.startswith('/StorageNode/Register/'):
       #this is a storage node register
       # syntax /Register/ipaddr:port/
       #print(_segmentLocator.get_statistics())
       _ipaddr=_path_info[22:]
       #print(_ipaddr)
       _segmentLocator.refresh(_ipaddr)
       start_response('200 OK', [('Content-type', 'text/plain')])
       return [''.encode('utf-8')]
    #elif _path_info.startswith('/RegisterMetadataNode/'):
    #   #this is a storage node register
    #   # syntax /Register/ipaddr:port/
    #   _ipaddr=_path_info[22:]
    #   self._metadataNodes.refresh(_ipaddr)
    elif _path_info.startswith('/Monitor'):
       #return a nice looking page with basic info such as
       #the number of storage nodes, the number of segments
       #each is storing, the last time they checked in, etc

       _fp=open("templates/default.tmpl", "r")
       _nodes_dict=_segmentLocator.get_statistics()
       _storagenodes=[]
       _mdn=[]
       _missing_segments=[]
       for _node in _nodes_dict.keys():
           _data=_nodes_dict[_node]
           _data['timedelta']=misc.convert_delta_time(time.time() - _data['timestamp'])
           _data['timestamp']=misc.convert_seconds(_data['timestamp'])
           _data['free']=misc.convert_bytes(_data['free'])
           _data['used']=misc.convert_bytes(_data['used'])
           
           _storagenodes.append(_nodes_dict[_node])

       _data={'last_updated': datetime.datetime.now(),
              'storagenodes': _storagenodes,
              'metadatanodes': _mdn,
              'missingsegments': _fsmetadata.get_statistics()}
       _out=io.StringIO()
       _tmpl=XYAPTU.xcopier(_data, ouf=_out)
       _tmpl.xcopy(_fp)
       start_response('200 OK', [('Content-type', 'text/html')])
       return [_out.getvalue().encode('utf-8')]
    elif _path_info.startswith('/Client'):
       #this is from a client that will want certain information
       #command list (listdir, rmdir, mkdir)
       # (put file, get file)
       _cmd=_path_info[7:]

       if _cmd.startswith('/locate_segments'):
          _result=misc.get_wsgi_file_contents(environ)
          _segments=misc.receive_compressed_response(_result)
          #_segments=json.loads(_result.decode('utf-8'))
          _dict=_segmentLocator.locate_segments(_segments)

          start_response('200 OK', [('Content-type', 'text/plain')])
          return [misc.send_compressed_response(_dict)]
       elif _cmd.startswith('/listdir/'):
          _id=_cmd[9:]
          #print(_id)
          _result=_fsmetadata.listdir(_id)
          if _result is None:
             start_response('404 OK', [('Content-type', 'text/plain')])
          else:
             start_response('200 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/Name2ID/'):
          _name=_cmd[9:]
          #print(_name)
          _result=_fsmetadata.lookupID(_name)
          print(_result)
          if _result is None:
             start_response('404 OK', [('Content-type', 'text/plain')])
          else:
             start_response('200 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/rmdir/'):
          _parentID, _id, _name=_cmd[7:].split('/',2)
          _result=_fsmetadata.rmdir(_parentID, _id, _name)
          start_response('200 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/cp/'):
            #copy metadata within the grid (ie, from grid location to grid location)
          _parentID, _shaID, _target_dir_id, _name=_cmd[4:].split('/',3)
          _result=_fsmetadata.cp(_parentID, _shaID, _target_dir_id, _name)
          start_response('200 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/mv/'):
            #copy metadata within grid
          _parentID, _shaID, _target_dir_id, _name=_cmd[4:].split('/',3)
          _result=_fsmetadata.cp(_parentID, _shaID, _target_dir_id, _name)
          start_response('200 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/mkdir/'):
          _parentid, _name_id, _name=_cmd[7:].split('/', 2)
          print(_parentid, _name_id, _name)
          _result=_fsmetadata.mkdir(_parentid, _name_id, _name)
          if _result is None:
             start_response('200 OK', [('Content-type', 'text/plain')])
             _result=''
          else:
             print(_result)
             start_response('404 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/putfile/'):
          _parentID, _file_name=_cmd[9:].split('/',1)
          _file_metadata=misc.get_wsgi_file_contents(environ)
          #get file metadata
          if _file_metadata is not None:
             _shaID=misc.get_shaID(_file_metadata)
             _size=0

             _result=_fsmetadata.putfile(_parentID, _shaID, _file_name, _size)
             #print(_result)
             if _result is None:
                #_metadata=json.loads(_file_metadata.decode('utf-8'))
                _fsmetadata.add_file_metadata(_file_metadata)
                start_response('200 OK', [('Content-type', 'text/plain')])
                return [json.dumps(_result).encode('utf-8')]
          print(_result)
          _result=''
          start_response('404 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_result).encode('utf-8')]
       elif _cmd.startswith('/getfile/'):
          _shaID=_cmd[9:]
          _valid, _result=_fsmetadata.get_file_metadata(_shaID)
          if _result is None:
             start_response('404 OK', [('Content-type', 'text/plain')])
             _result=''
          else:
             start_response('200 OK', [('Content-type', 'text/plain')])

          return [misc.send_compressed_response(_result.decode('utf-8'))] 
          #return [json.dumps(_result).encode('utf-8')] 
       elif _cmd.startswith('/getStorageNodeStatistics'):
          _dict=_segmentLocator.get_statistics()
          if _dict=={}:  #try refreshing to get a list of nodes
             _segmentLocator.refresh()
             _dict=_segmentLocator.get_statistics()

          #send string to client
          start_response('200 OK', [('Content-type', 'text/plain')])
          return [json.dumps(_dict).encode('utf-8')]
       elif _cmd.startswith('/rmfile/'):
          _dir_id, _shaID=_cmd[8:].split('/')

          _result=_fsmetadata.rmfile(_dir_id, _shaID)
          #if _fsmetadata.get_file_locations(_shaID)==[]:
             #this file isn't in the metadata anymore so lets 
             #remove the segments
             #_fsmetadata.rmfile(_shaID)

          if _result is None:
             start_response('200 OK', [('Content-type', 'text/plain')])
             return [json.dumps('').encode('utf-8')]
          else:
             start_response('404 OK', [('Content-type', 'text/plain')])
             return [json.dumps(_dict).encode('utf-8')]

       start_response('404 OK', [('Content-type', 'text/plain')])
       return [json.dumps('').encode('utf-8')]