def _load(self):
try:
results = DBFactory.get_db().get(self.guid)
for key in results:
val = results[key]
if isinstance(val, basestring) and val.startswith('DICTABLE'):
i = val.index('|', 9)
smod, sclass = val[9:i].split(':')
value = unpack(val[i+1:])
module = __import__(smod, fromlist=[sclass])
classobj = getattr(module, sclass)
value = classobj._fromdict(value)
elif key in ('root_dir', 'file_path'):
# No op - set in constructor
continue
elif key == 'brick_tree':
setattr(self, key, RTreeProxy.deserialize(val))
continue
elif key == 'span_collection':
unpacked = unpack(val)
value = SpanCollectionByFile.from_str(unpacked)
log.trace("Reconstructed SpanCollection for %s: %s", self.guid, str(value))
else:
value = unpack(val)
if isinstance(value, tuple):
value = list(value)
setattr(self, key, value)
except Exception as e:
log.error("Caught exception reconstructing metadata for guid %s : %s", self.guid, e.message)
raise
def decompress(self, obj):
vals = base64decode(obj)
if isinstance(vals, np.ndarray):
return NumpyParameterData(self.parameter_manager.parameter_name, vals)
# if isinstance(vals[0], basestring) and vals[0].startswith('_mp_'):
#
# st = vals[0][len('_mp_'):]
# if hasattr(vals[0], '__iter__'):
# vals = np.array([unpack(x) for x in vals[0]])
# else:
# vals = np.array([unpack(vals[0])])
# return NumpyParameterData(self.parameter_manager.parameter_name, vals)
elif isinstance(vals, tuple) and len(vals) == 1 and isinstance(vals[0], basestring):
# if hasattr(vals[0], '__iter__'):
# vals = np.array([unpack(x) for x in vals[0]])
# else:
vals = np.array([unpack(vals[0])])
return NumpyParameterData(self.parameter_manager.parameter_name, vals)
elif isinstance(vals, tuple):
if vals[4] == ConstantOverTime.__name__:
return ConstantOverTime(self.parameter_manager.parameter_name, vals[1], time_start=vals[2], time_end=vals[3])
elif vals[4] == RepeatOverTime.__name__:
return RepeatOverTime(self.parameter_manager.parameter_name, vals[1], time_start=vals[2], time_end=vals[3])
else:
raise Exception("Could not rebuild class %s" % vals[4])
else:
raise Exception("Could not handle decompressed value %s" % vals)
def getCoverageType(directory, guid):
cov_type = DBFactory.get_db().get_coverage_type(guid)
if '' == cov_type:
return ''
else:
cov_type = unpack(cov_type)
return cov_type
def _object_unpack_hook(self, value):
value = unpack(value)
if isinstance(value, np.ndarray):
return value.tolist()
elif isinstance(value, tuple):
return list(value)
else:
return value
def _object_unpack_hook(self, value):
value = unpack(value)
if isinstance(value, np.ndarray):
return value.tolist()
elif isinstance(value, tuple):
return list(value)
else:
return value
def __getitem__(self, slice_):
if not isinstance(slice_, (list,tuple)):
slice_ = [slice_]
log.debug('getitem slice_: %s', slice_)
arr_shp = self._get_array_shape_from_slice(slice_)
ret_arr = np.empty(arr_shp, dtype=self.dtype)
ret_arr.fill(self.fill_value)
ret_origin = [0 for x in range(ret_arr.ndim)]
log.trace('Shape of returned array: %s', ret_arr.shape)
if arr_shp == 0:
return ret_arr
brick_origin_offset = 0
bricks = self._bricks_from_slice(slice_)
log.trace('Slice %s indicates bricks: %s', slice_, bricks)
for idx, brick_guid in bricks:
brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, brick_guid)
# Figuring out which part of brick to set values - also appropriately increments the ret_origin
log.trace('Return array origin: %s', ret_origin)
try:
brick_slice, value_slice, brick_origin_offset = self._calc_slices(slice_, brick_guid, ret_arr, ret_origin, brick_origin_offset)
if brick_slice is None:
raise ValueError('Brick contains no values for specified slice')
except ValueError as ve:
log.warn(ve.message + '; moving to next brick')
continue
log.trace('Brick slice to extract: %s', brick_slice)
log.trace('Value slice to fill: %s', value_slice)
if not os.path.exists(brick_file_path):
log.trace('Found virtual brick file: %s', brick_file_path)
else:
log.trace('Found real brick file: %s', brick_file_path)
with h5py.File(brick_file_path) as brick_file:
v = brick_file[brick_guid].__getitem__(*brick_slice)
# Check if object type
if self.dtype == '|O8':
if not hasattr(v, '__iter__'):
v = [v]
v = [unpack(x) for x in v]
ret_arr[value_slice] = v
if ret_arr.size == 1:
if ret_arr.ndim==0:
ret_arr=ret_arr[()]
else:
ret_arr=ret_arr[0]
return ret_arr
def __deserialize(self, payload):
if isinstance(payload, basestring) and payload.startswith('DICTABLE'):
i = payload.index('|', 9)
smod, sclass = payload[9:i].split(':')
value = unpack(payload[i + 1:])
module = __import__(smod, fromlist=[sclass])
classobj = getattr(module, sclass)
payload = classobj._fromdict(value)
return payload
def __deserialize(self, payload):
if isinstance(payload, basestring) and payload.startswith('DICTABLE'):
i = payload.index('|', 9)
smod, sclass = payload[9:i].split(':')
value = unpack(payload[i + 1:])
module = __import__(smod, fromlist=[sclass])
classobj = getattr(module, sclass)
payload = classobj._fromdict(value)
return payload
def getCoverageType(directory, guid):
if DbBackedMetadataManager.is_persisted_in_db(guid) is True:
cov_type = DBFactory.get_db().get_coverage_type(guid)
if '' == cov_type:
return ''
else:
cov_type = unpack(cov_type)
return cov_type
else:
return BaseManager.getCoverageType(directory, guid)
def from_msgpack(cls, str, decompressors=None):
from coverage_model.persistence_helpers import unpack
_dict = unpack(str)
uncompressed_params = {}
if 'params' in _dict:
for param, data in _dict['params'].iteritems():
if decompressors is not None:
uncompressed_params[param] = decompressors[param].decompress(data)
else:
uncompressed_params[param] = np.array(data[0], dtype=data[1])
uncompressed_params[param].shape = data[2]
span = Span(_dict['id'], _dict['coverage_id'], uncompressed_params, ingest_time=_dict['ingest_time'], mutable=_dict['mutable'])
span.ingest_times = _dict['ingest_time_dict']
span.param_dict[cls.ingest_time_str] = NumpyParameterData(cls.ingest_time_str, span.ingest_time_array)
return span
def from_pack(text):
pack_dict = msgpack.unpackb(text)
guid = pack_dict.pop('guid')
parameter_name = pack_dict.pop('parameter_name')
pm = ParameterContextWrapper(guid, parameter_name, read_only=True)
for k, val in pack_dict.iteritems():
if isinstance(val, basestring) and val.startswith('DICTABLE'):
i = val.index('|', 9)
smod, sclass = val[9:i].split(':')
val = unpack(val[i+1:])
module = __import__(smod, fromlist=[sclass])
classobj = getattr(module, sclass)
val = classobj._fromdict(val)
pm.__setattr__(k,val)
return pm
def __getitem__(self, slice_):
# Always storing in first slot - ignore slice
if len(self.brick_list) == 0:
raise ValueError('No Bricks!')
bid = 'sparse_value_brick'
brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid)
if os.path.exists(brick_file_path):
with HDFLockingFile(brick_file_path) as f:
ret_vals = f[bid][0]
else:
ret_vals = None
if ret_vals is None:
return self.fill_value
ret_vals = unpack(ret_vals)
ret = [self.__deserialize(v) for v in ret_vals]
return ret
def __getitem__(self, slice_):
# Always storing in first slot - ignore slice
if len(self.brick_list) == 0:
raise ValueError('No Bricks!')
bid = 'sparse_value_brick'
brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid)
if os.path.exists(brick_file_path):
with h5py.File(brick_file_path) as f:
ret_vals = f[bid][0]
else:
ret_vals = None
if ret_vals is None:
return self.fill_value
ret_vals = unpack(ret_vals)
ret = [self.__deserialize(v) for v in ret_vals]
return ret