def __init__( self, fn, mode ):    

    Frontend.__init__( self, fn, mode );

    self._kdbfn = None;
    self._kdb = None;

    self._ldbdn = None;
    self._ldb = None;

    self._len_c = None;
    self._len_b = None;
    self._len_x = None;

    self._ic = None;
    self._icbp = None;

    self._needs_initialization = True;

    self._core_dims = set();
    self._satellite_dims = set();
    self._removed_dims = set();

    self._remove_c = set();
    self._remove_b = set();
    self._remove_x = set();

    self.bypass_c = False;
    self.bypass_b = False;
    self.bypass_x = False;
  def __init__( self, fn, mode ):

    Frontend.__init__( self, fn, mode );
    self._max_rows = 50000;

    self._dbs = {};
    self._dbdirs = [];
  def _finalize( self ):

    assert Frontend._finalize( self ) is None;

    rowids_for_ntile_bounds = [];
    for i in range( 1, 32 ):
      rowids_for_ntile_bounds.append(
          int( float(self._rowcount) * float(i)/32.0 )
        );

    ntile_bound_by_dim = {};
    for i in self._dbs:
      ntile_bound_by_dim[ i ] = [];

    for ( i, db ) in self._dbs.items():

      with db.iterator() as it:

        rowid = 0;

        for ( valid, ( xval, xcnt ) ) in enumerate( it ):

          xval = unpack( '>I', xval )[ 0 ] - (1<<31);
          xcnt = unpack( '>I', xcnt )[ 0 ];

          rowid_old = rowid;
          rowid = rowid_old + xcnt;

          for bound in rowids_for_ntile_bounds:
            if rowid_old < bound <= rowid:
              ntile_bound_by_dim[ i ].append( xval );

    self._state = ntile_bound_by_dim;
  def train( self, row ):

    if Frontend.train( self, row ):
      return True;

    for ( i, xval ) in enumerate( row ):

      if not i in self._dbs:
        dbdn = None;
        with TemporaryDirectory() as tmpdirname:
          dbdn = tmpdirname;        
        self._dbdirs.append( dbdn );
        self._dbs[ i ] = plyvel.DB( dbdn, create_if_missing=True );

      xval \
        = int( float(xval) * 1000.0 );

      assert xval < (1<<30);

      xval \
        = pack( ">I", (1<<31) + xval );

      xcnt \
        = unpack(
              ">I",
              self._dbs[ i ].get( xval, default=pack( ">I", 0 ) )
            )[ 0 ];

      xcnt += 1;

      self._dbs[ i ].put( xval, pack( ">I", xcnt ) );

    return False;
  def train( self, row ):

    if Frontend.train( self, row ):
      return True;

    self._data.append( row );

    return False;
  def __exit__( self, exc_type, exc_value, traceback ):

    assert Frontend.__exit__( self, exc_type, exc_value, traceback ) == False;

    sleep( 3.0 );

    for db in self._dbs.values():
      db.close();    
    for dn in self._dbdirs:
      rmtree( dn );

    return False;
  def train( self, row ):

    if Frontend.train( self, row ):
      return True;

    b = 0;
    for i in range( 0, len(row) ):
      if row[i] == 1:
        b |= (1<<i);

    self._stats[ b ] = self._stats.get( b, 0 ) + 1;

    return False;
  def _finalize( self ):

    assert Frontend._finalize( self ) is None;

    data = np.array( self._data ).T;

    rest = [];
    for i in range( 1, len(data)+1 ):
      rest.append( {i} );

    while True:
      len_rest_before = len(rest);
      rest = self._merge( data, rest );    
      # print( rest );
      if len( rest ) == len_rest_before:
        break;

    self._state = rest;

    if False:

      print( "-- CLUSTERS --" );

      print( len(rest), rest );

    if False:

      print( "-- INSIDE CORRELATIONS --" );

      for dc in rest:
        print( dc );
        for di in dc:
          for dj in dc:
            if abs(di) >= abs(dj):
              continue;
            corr = np.corrcoef( data[abs(di)-1], data[abs(dj)-1] )[ 0 ][ 1 ];
            print( di, dj, corr );

    if False:

      print( "-- OUTSIDE CORRELATIONS --" );

      for ( i, dci ) in enumerate( rest ):
        ivals = self._cluster_val( data, dci );
        for ( j, dcj ) in enumerate( rest ):
          if i >= j:
            continue;
          jvals = self._cluster_val( data, dcj );
          corr = np.corrcoef( ivals, jvals )[ 0 ][ 1 ];
          print( i, j, corr );
  def __call__( self, row ):

    assert Frontend.__call__( self, row ) is row;

    val = 0;
    for (i,row_i) in enumerate( row ):
      if row_i == 1:
        val |= (1<<i);

    row_ = [];

    for cluster in self._state:
      mask = 0;
      for dim in cluster:
        mask |= (1<<dim);
      row_.append( val & mask );

    return row_;
  def __exit__( self, exc_type, exc_value, traceback ):

    assert Frontend.__exit__( self, exc_type, exc_value, traceback ) == False;

    if self._ldb is not None:
      sleep( 3.0 );
      self._ldb.close()

    if self._ldbdn is not None:
      rmtree( self._ldbdn );

    if self._kdb is not None:
      try:
        assert self._kdb.close();
      except:
        print( str( self._kdb.error() ) );
        raise;

    if self._kdbfn is not None:
      remove( self._kdbfn );
  def __call__( self, row ):

    assert Frontend.__call__( self, row ) is row;

    row_ = [];

    for cluster in self._state:

      val = 0.0;

      for dim in cluster:
        if dim > 0:
          val += row[dim];
        else:
          assert dim < 0;
          val -= row[dim];

      val /= len(cluster);

      row_.append( val );

    return row_;
  def train( self, row ):

    ( y, c, b, x ) = row;

    if self._len_c is None:
      self._len_c = len(c);
    assert self._len_c == len(c);

    if self._len_b is None:
      self._len_b = len(b);
    assert self._len_b == len(b);

    if self._len_x is None:
      self._len_x = len(x);
    assert self._len_x == len(x);

    row = c + b + x;

    if Frontend.train( self, row ):
      return True;

    keyfmt = '>IIIII';

    for i in range( 0, self._lenrow ):
      for j in range( 0, self._lenrow ):

        if ( i >= j ) and ( not ( i == self._lenrow-1 ) ):
          continue;

        key = pack( keyfmt, i, j, y, row[i], row[j] );

        try:
          assert self._kdb.increment( key, 1, 0 );
        except:
          print( str(self._kdb.error()) );
          raise;
  def _finalize( self ):

    assert Frontend._finalize( self ) is None;

    self._state = [];
    rest = set( range( 0, self._lenrow ) );

    while rest:
      if len( rest ) >= 2:
        ( left, right, rest ) = self._split( rest );
        self._state.append( left );
        self._state.append( right );
      else:
        self._state.append( rest );
        rest = set();

    if False:

      for ( cluster_id, cluster ) in enumerate( self._state ):
        print( "-->", cluster_id, cluster );
        for i in cluster:
          a = { i };
          b = cluster - a;
          print( "    {:d} {:1.4f}".format( i, self._i_corr( a, b ) ) );
  def _finalize( self ):

    assert Frontend._finalize( self ) is None;

    if False:
      print( "unique combinations = ", self._kdb.count() );

    keyfmt = '>IIIII';
    valfmt = '>Q';

    c = self._kdb.cursor();
    c.jump();

    gt2 = 0;
    gt4 = 0;
    gt8 = 0;
    gt16 = 0;
    gt32 = 0;

    while True:

      r = c.get( True );
      if not r:
        break;

      self._ldb.put( r[0], r[1] );

      key = unpack( keyfmt, r[0] );
      val = unpack( valfmt, r[1] )[ 0 ];

      if val > 2:
        gt2 += 1;
      if val > 4:
        gt4 += 1;
      if val > 8:
        gt8 += 1;
      if val > 16:
        gt16 += 1;
      if val > 32:
        gt32 += 1;

    if False:
      print( gt2, gt4, gt8, gt16, gt32 );

    self._ic = {};
    for i in range( 0, self._lenrow ):
      self._ic[ i ] = self._get_info_content_by_dimension( i );

    self._icbp = {};

    for i in range( 0, self._lenrow ):
      for j in range( 0, self._lenrow ):

        if i >= j:
          continue;

        self._icbp[ (i,j) ] = self._get_info_content_by_pair( i, j );


    self._state \
      = { "ic": self._ic,
          "icbp": self._icbp,
          "c": self._len_c,
          "b": self._len_b,
          "x": self._len_x };
  def __init__( self, fn, mode ):

    Frontend.__init__( self, fn, mode );
    self._max_rows = 100000;
    
    self._stats = {};
  def __init__( self, fn, mode ):

    Frontend.__init__( self, fn, mode );
    self._max_rows = 100000;

    self._data = [];