Ejemplo n.º 1
0
    def save(self, data):
        deferredsave = []
        pipe = self.pipeline()
        for key, value in data.items():
            if key not in self.schema.keys():
                deferredsave.append(key)
                continue

            if self.schema[key] == 'list':
                pipe.delete(key)
                for val in value:
                    pipe.rpush(key, val)
            elif self.schema[key] == 'dict':
                pipe.hmset(key, value)
            elif self.schema[key] in ['matrix', 'ndarray']:
                deferredsave.append(key)
            else:
                pipe.set(key, value)
            logging.debug("  Saving data elm  `%s` as %s ", key,
                          type(data[key]))

        result = pipe.execute()

        for key in deferredsave:
            if key not in self.schema.keys():
                if isinstance(data[key], list):
                    self.delete(key)
                    for elm in data[key]:
                        self.rpush(key, elm)
                elif isinstance(data[key], dict):
                    self.hmset(key, data[key])
                else:
                    self.set(key, data[key])
                logging.debug("Dynamically saving %s. Updating schema", key)

            elif self.schema[key] == 'ndarray':
                try:
                    self.storeNPArray(data[key])
                except TypeError as err:
                    print('KEY:', key)
                    raise
            elif self.schema[key] == 'matrix':
                if isinstance(data[key], np.ndarray):
                    matrix = kv2DArray(self, key)
                    matrix.set(data[key])
                else:
                    matrix = kv2DArray(
                        self,
                        key,
                    )

        return result
Ejemplo n.º 2
0
  def save(self, data):
    deferredsave = []
    pipe = self.pipeline()
    for key, value in data.items():
      if key not in self.schema.keys():
        deferredsave.append(key)
        continue

      if self.schema[key] == 'list':
        pipe.delete(key)
        for val in value:
          pipe.rpush(key, val)
      elif self.schema[key] == 'dict':
        pipe.hmset(key, value)
      elif self.schema[key] in ['matrix', 'ndarray']:
        deferredsave.append(key)
      else:
        pipe.set(key, value)
      logging.debug("  Saving data elm  `%s` as %s ", key, type(data[key]))

    result = pipe.execute()

    for key in deferredsave:
      if key not in self.schema.keys():
        if isinstance(data[key], list):
          self.delete(key)
          for elm in data[key]:
            self.rpush(key, elm)
        elif isinstance(data[key], dict):
          self.hmset(key, data[key])
        else:
          self.set(key, data[key])
        logging.debug("Dynamically saving %s. Updating schema", key)

      elif self.schema[key] == 'ndarray':
        try:
          self.storeNPArray(data[key])
        except TypeError as err:
          print('KEY:', key)
          raise
      elif self.schema[key] == 'matrix':
        if isinstance(data[key], np.ndarray):
          matrix = kv2DArray(self, key)
          matrix.set(data[key])
        else:
          matrix = kv2DArray(self, key, )

    return result
Ejemplo n.º 3
0
    def append(self, data):
        logging.debug('CATALOG APPEND: %s', str(data.keys()))
        deferredappend = []
        try:
            self.check_connection()
            pipe = self.pipeline()
            for key, value in data.items():
                logging.debug("Appending data elm  `%s` of type, %s", key,
                              type(data[key]))
                if key not in self.schema.keys():
                    logging.warning(
                        "  KEY `%s` not found in local schema! Will try Dynamic Append"
                    )
                    deferredappend.append(key)
                elif self.schema[key] == 'int':
                    pipe.incr(key, value)
                elif self.schema[key] == 'float':
                    pipe.incrbyfloat(key, value)
                elif self.schema[key] == 'list':
                    for val in value:
                        pipe.rpush(key, val)
                elif self.schema[key] == 'dict':
                    for k, v in value:
                        pipe.hset(key, k, v)
                elif self.schema[key] in ['matrix', 'ndarray']:
                    deferredappend.append(key)
                else:
                    pipe.set(key, value)

            result = pipe.execute()

            for key in deferredappend:
                if self.schema[key] == 'ndarray':
                    logging.warning(" NUMPY ARRAY MERGING NOT SUPPORTED")
                    # self.storeNPArray(data[key])
                elif self.schema[key] == 'matrix':
                    logging.warning("  Merging matrix `%s`:  %s", key,
                                    str(data[key]))
                    matrix = kv2DArray(self, key)
                    matrix.merge(data[key])
            return result
        except OverlayNotAvailable as e:
            logging.error(
                'Redis Server is not available. Cannot run pipeline command')
            raise OverlayNotAvailable
Ejemplo n.º 4
0
  def append(self, data):
    logging.debug('CATALOG APPEND: %s', str(data.keys()))
    deferredappend = []
    try:
      self.check_connection()
      pipe = self.pipeline()
      for key, value in data.items():
        logging.debug("Appending data elm  `%s` of type, %s", key, type(data[key]))
        if key not in self.schema.keys():
          logging.warning("  KEY `%s` not found in local schema! Will try Dynamic Append")
          deferredappend.append(key)
        elif self.schema[key] == 'int':
          pipe.incr(key, value)
        elif self.schema[key] == 'float':
          pipe.incrbyfloat(key, value)
        elif self.schema[key] == 'list':
          for val in value:
            pipe.rpush(key, val)
        elif self.schema[key] == 'dict':
          for k, v in value:
            pipe.hset(key, k, v)
        elif self.schema[key] in ['matrix', 'ndarray']:
          deferredappend.append(key)
        else:
          pipe.set(key, value)

      result = pipe.execute()

      for key in deferredappend:
        if self.schema[key] == 'ndarray':
          logging.warning(" NUMPY ARRAY MERGING NOT SUPPORTED")
          # self.storeNPArray(data[key])
        elif self.schema[key] == 'matrix':
          logging.warning("  Merging matrix `%s`:  %s", key, str(data[key]))
          matrix = kv2DArray(self, key)
          matrix.merge(data[key])
      return result
    except OverlayNotAvailable as e:
      logging.error('Redis Server is not available. Cannot run pipeline command')
      raise OverlayNotAvailable
Ejemplo n.º 5
0
  def load(self, keylist):

    if isinstance(keylist, dict):
      logging.error("NOT ACCEPTING DICT HERE")
      sys.exit(0)

    keys = keylist if isinstance(keylist, list) else [keylist]
    data = {}
    deferredload = []

    # Pipeline native data type
    pipe = self.pipeline()
    for key in keys:
      if key not in self.schema.keys():
        deferredload.append(key)
        continue
      if self.schema[key] == 'list':
        pipe.lrange(key, 0, -1)
      elif self.schema[key] == 'dict':
        pipe.hgetall(key)
      elif self.schema[key] in ['matrix', 'ndarray']:
        deferredload.append(key)
      else:
        pipe.get(key)

    vals = pipe.execute()

    #  Deferred load for retrieval of non-native data types
    for key in deferredload:
      if key not in self.schema.keys():
        try:
          value = self.lrange(key, 0, -1)
        except redis.ResponseError as ex:
          try:
            value = self.hgetall(key)
          except redis.ResponseError as ex:
            value = self.get(key)
        data[key] = decodevalue(value)
        self.schema[key] = type(value).__name__

      elif self.schema[key] == 'ndarray':
        data[key] = self.loadNPArray(key)
      
      elif self.schema[key] == 'matrix':
        matrix = kv2DArray(self, key)
        data[key] = matrix.get()

    #  Data Conversion
    # TODO:  MOVE TO JSON or pickel BASED RETRIEVAL (???)

    i = 0

    for key in keys:
      try:
        if key in deferredload:
          continue
        data[key] = decodevalue(vals[i])
        if data[key] is None:
          if self.schema[key] == 'list':
            data[key] = []
          elif self.schema[key] == 'dict':
            data[key] = {}
        i += 1
      except (AttributeError, KeyError) as ex:
        logging.error("ERROR! Loading a BAD KEY:  %s", key)
        logging.error("Trace:  %s", str(ex))
        sys.exit(0)

    return data
Ejemplo n.º 6
0
    def load(self, keylist):

        if isinstance(keylist, dict):
            logging.error("NOT ACCEPTING DICT HERE")
            sys.exit(0)

        keys = keylist if isinstance(keylist, list) else [keylist]
        data = {}
        deferredload = []

        # Pipeline native data type
        pipe = self.pipeline()
        for key in keys:
            if key not in self.schema.keys():
                deferredload.append(key)
                continue
            if self.schema[key] == 'list':
                pipe.lrange(key, 0, -1)
            elif self.schema[key] == 'dict':
                pipe.hgetall(key)
            elif self.schema[key] in ['matrix', 'ndarray']:
                deferredload.append(key)
            else:
                pipe.get(key)

        vals = pipe.execute()

        #  Deferred load for retrieval of non-native data types
        for key in deferredload:
            if key not in self.schema.keys():
                try:
                    value = self.lrange(key, 0, -1)
                except redis.ResponseError as ex:
                    try:
                        value = self.hgetall(key)
                    except redis.ResponseError as ex:
                        value = self.get(key)
                data[key] = decodevalue(value)
                self.schema[key] = type(value).__name__

            elif self.schema[key] == 'ndarray':
                data[key] = self.loadNPArray(key)

            elif self.schema[key] == 'matrix':
                matrix = kv2DArray(self, key)
                data[key] = matrix.get()

        #  Data Conversion
        # TODO:  MOVE TO JSON or pickel BASED RETRIEVAL (???)

        i = 0

        for key in keys:
            try:
                if key in deferredload:
                    continue
                data[key] = decodevalue(vals[i])
                if data[key] is None:
                    if self.schema[key] == 'list':
                        data[key] = []
                    elif self.schema[key] == 'dict':
                        data[key] = {}
                i += 1
            except (AttributeError, KeyError) as ex:
                logging.error("ERROR! Loading a BAD KEY:  %s", key)
                logging.error("Trace:  %s", str(ex))
                sys.exit(0)

        return data