def save(self, data): deferredsave = [] pipe = self.pipeline() for key, value in data.items(): if key not in self.schema.keys(): deferredsave.append(key) continue if self.schema[key] == 'list': pipe.delete(key) for val in value: pipe.rpush(key, val) elif self.schema[key] == 'dict': pipe.hmset(key, value) elif self.schema[key] in ['matrix', 'ndarray']: deferredsave.append(key) else: pipe.set(key, value) logging.debug(" Saving data elm `%s` as %s ", key, type(data[key])) result = pipe.execute() for key in deferredsave: if key not in self.schema.keys(): if isinstance(data[key], list): self.delete(key) for elm in data[key]: self.rpush(key, elm) elif isinstance(data[key], dict): self.hmset(key, data[key]) else: self.set(key, data[key]) logging.debug("Dynamically saving %s. Updating schema", key) elif self.schema[key] == 'ndarray': try: self.storeNPArray(data[key]) except TypeError as err: print('KEY:', key) raise elif self.schema[key] == 'matrix': if isinstance(data[key], np.ndarray): matrix = kv2DArray(self, key) matrix.set(data[key]) else: matrix = kv2DArray( self, key, ) return result
def save(self, data): deferredsave = [] pipe = self.pipeline() for key, value in data.items(): if key not in self.schema.keys(): deferredsave.append(key) continue if self.schema[key] == 'list': pipe.delete(key) for val in value: pipe.rpush(key, val) elif self.schema[key] == 'dict': pipe.hmset(key, value) elif self.schema[key] in ['matrix', 'ndarray']: deferredsave.append(key) else: pipe.set(key, value) logging.debug(" Saving data elm `%s` as %s ", key, type(data[key])) result = pipe.execute() for key in deferredsave: if key not in self.schema.keys(): if isinstance(data[key], list): self.delete(key) for elm in data[key]: self.rpush(key, elm) elif isinstance(data[key], dict): self.hmset(key, data[key]) else: self.set(key, data[key]) logging.debug("Dynamically saving %s. Updating schema", key) elif self.schema[key] == 'ndarray': try: self.storeNPArray(data[key]) except TypeError as err: print('KEY:', key) raise elif self.schema[key] == 'matrix': if isinstance(data[key], np.ndarray): matrix = kv2DArray(self, key) matrix.set(data[key]) else: matrix = kv2DArray(self, key, ) return result
def append(self, data): logging.debug('CATALOG APPEND: %s', str(data.keys())) deferredappend = [] try: self.check_connection() pipe = self.pipeline() for key, value in data.items(): logging.debug("Appending data elm `%s` of type, %s", key, type(data[key])) if key not in self.schema.keys(): logging.warning( " KEY `%s` not found in local schema! Will try Dynamic Append" ) deferredappend.append(key) elif self.schema[key] == 'int': pipe.incr(key, value) elif self.schema[key] == 'float': pipe.incrbyfloat(key, value) elif self.schema[key] == 'list': for val in value: pipe.rpush(key, val) elif self.schema[key] == 'dict': for k, v in value: pipe.hset(key, k, v) elif self.schema[key] in ['matrix', 'ndarray']: deferredappend.append(key) else: pipe.set(key, value) result = pipe.execute() for key in deferredappend: if self.schema[key] == 'ndarray': logging.warning(" NUMPY ARRAY MERGING NOT SUPPORTED") # self.storeNPArray(data[key]) elif self.schema[key] == 'matrix': logging.warning(" Merging matrix `%s`: %s", key, str(data[key])) matrix = kv2DArray(self, key) matrix.merge(data[key]) return result except OverlayNotAvailable as e: logging.error( 'Redis Server is not available. Cannot run pipeline command') raise OverlayNotAvailable
def append(self, data): logging.debug('CATALOG APPEND: %s', str(data.keys())) deferredappend = [] try: self.check_connection() pipe = self.pipeline() for key, value in data.items(): logging.debug("Appending data elm `%s` of type, %s", key, type(data[key])) if key not in self.schema.keys(): logging.warning(" KEY `%s` not found in local schema! Will try Dynamic Append") deferredappend.append(key) elif self.schema[key] == 'int': pipe.incr(key, value) elif self.schema[key] == 'float': pipe.incrbyfloat(key, value) elif self.schema[key] == 'list': for val in value: pipe.rpush(key, val) elif self.schema[key] == 'dict': for k, v in value: pipe.hset(key, k, v) elif self.schema[key] in ['matrix', 'ndarray']: deferredappend.append(key) else: pipe.set(key, value) result = pipe.execute() for key in deferredappend: if self.schema[key] == 'ndarray': logging.warning(" NUMPY ARRAY MERGING NOT SUPPORTED") # self.storeNPArray(data[key]) elif self.schema[key] == 'matrix': logging.warning(" Merging matrix `%s`: %s", key, str(data[key])) matrix = kv2DArray(self, key) matrix.merge(data[key]) return result except OverlayNotAvailable as e: logging.error('Redis Server is not available. Cannot run pipeline command') raise OverlayNotAvailable
def load(self, keylist): if isinstance(keylist, dict): logging.error("NOT ACCEPTING DICT HERE") sys.exit(0) keys = keylist if isinstance(keylist, list) else [keylist] data = {} deferredload = [] # Pipeline native data type pipe = self.pipeline() for key in keys: if key not in self.schema.keys(): deferredload.append(key) continue if self.schema[key] == 'list': pipe.lrange(key, 0, -1) elif self.schema[key] == 'dict': pipe.hgetall(key) elif self.schema[key] in ['matrix', 'ndarray']: deferredload.append(key) else: pipe.get(key) vals = pipe.execute() # Deferred load for retrieval of non-native data types for key in deferredload: if key not in self.schema.keys(): try: value = self.lrange(key, 0, -1) except redis.ResponseError as ex: try: value = self.hgetall(key) except redis.ResponseError as ex: value = self.get(key) data[key] = decodevalue(value) self.schema[key] = type(value).__name__ elif self.schema[key] == 'ndarray': data[key] = self.loadNPArray(key) elif self.schema[key] == 'matrix': matrix = kv2DArray(self, key) data[key] = matrix.get() # Data Conversion # TODO: MOVE TO JSON or pickel BASED RETRIEVAL (???) i = 0 for key in keys: try: if key in deferredload: continue data[key] = decodevalue(vals[i]) if data[key] is None: if self.schema[key] == 'list': data[key] = [] elif self.schema[key] == 'dict': data[key] = {} i += 1 except (AttributeError, KeyError) as ex: logging.error("ERROR! Loading a BAD KEY: %s", key) logging.error("Trace: %s", str(ex)) sys.exit(0) return data