def test_locking_contention(self):
guid = utils.create_guid()
path = os.path.join(self.working_dir, '%s.h5' % guid)
f1 = HDFLockingFile(path, 'a')
try:
f2 = HDFLockingFile(path,'a')
raise AssertionError('Failed to raise IOError for exclusion lock')
except IOError as e:
if 'Resource temporarily unavailable' not in e.message:
raise
f1.close()
def write_brick(origin,bD,cD,coverageGUID,parameterName):
#global brickTree
log.debug('Writing brick for parameter {0}'.format(parameterName))
log.debug('Brick origin: {0}'.format(origin))
rootPath = 'test_data/{0}/{1}'.format(coverageGUID,parameterName)
# Create the root path if it does not exist
# TODO: Eliminate possible race condition
if not os.path.exists(rootPath):
os.makedirs(rootPath)
# Create a GUID for the brick
brickGUID = create_guid()
# Set HDF5 file and group
sugarFileName = '{0}.hdf5'.format(brickGUID)
sugarFilePath = '{0}/{1}'.format(rootPath,sugarFileName)
sugarFile = HDFLockingFile(sugarFilePath, 'w')
sugarGroupPath = '/{0}/{1}'.format(coverageGUID,parameterName)
sugarGroup = sugarFile.create_group(sugarGroupPath)
# TODO: Remove after the ParameterContext is passed to the function
parameterData = np.empty(bD, dtype='f')
# Create the HDF5 dataset that represents one brick
sugarCubes = sugarGroup.create_dataset('{0}'.format(brickGUID), bD, dtype=parameterData.dtype, chunks=cD)
# Close the HDF5 file that represents one brick
log.debug('Size Before Close: {0}'.format(os.path.getsize(sugarFilePath)))
sugarFile.close()
log.debug('Size After Close: {0}'.format(os.path.getsize(sugarFilePath)))
# TODO: Keep track of brick GUID and index in an R-tree somewhere
# Calculate the brick extents for the Rtree
# TODO: May not need this
brickMax = []
#brickExtents = []
for idx,val in enumerate(origin):
brickMax.append(bD[idx]+val)
#for val in itertools.izip(origin,brickMax):
# brickExtents.append(val)
#bE = np.array(brickExtents).flatten().tolist()
bE = list(origin)+brickMax
log.debug('Brick extents (rtree format): {0}'.format(bE))
# Insert into the Rtree
# TODO: This is not working yet. Need an id system and the location of the brick
brickTree.insert(1,bE,obj=brickGUID)
def cov2hdf(scov):
log.debug('Converting SimplexCoverage to HDF5 objects...')
rootPath = 'test_data'
masterFileName = '{0}_{1}.hdf5'.format('sugarMaster',scov.label)
masterFilePath = '{0}/{1}'.format(rootPath,masterFileName)
coverageName = scov.label
# Open HDf5 file for writing out the SimplexCoverage
sugarMaster = HDFLockingFile(masterFilePath, 'w')
sugarMasterGroup = sugarMaster.create_group('/{0}'.format(coverageName))
# Get parameter names, context and data from the coverage
parameterNames = scov.list_parameters()
# Create persistence objects in HDF5
# Loop through each parameter in the coverage
for parameterName in parameterNames:
# Brick and chunking size is based on the parameter data and is configurable
parameterData = scov.range_value[parameterName]
# TODO: Get the brick and cube size from a pre-calculated and assigned coverage parameter
sugarBrickSize = parameterData.shape
sugarCubeSizeList = list()
sugarCubeSize = tuple
# Calculate the chunking size
for idx, val in enumerate(parameterData.shape):
if val <= 10:
sugarCubeSizeList.append(1)
else:
sugarCubeSizeList.append(math.trunc(val/10))
# Convert list to tuple
sugarCubeSize = tuple(sugarCubeSizeList)
# TODO: Calculate the number of bricks required based on coverage extents and target brick size
sugarBricks = np.arange(10)
# TODO: Split up all parameter data into brick-sized arrays
for sugarBrick in sugarBricks:
sugarFileName = '{0}_{1}_Brick{2}.hdf5'.format(coverageName,parameterName,sugarBrick)
sugarFilePath = '{0}/{1}'.format(rootPath,sugarFileName)
sugarFile = HDFLockingFile(sugarFilePath, 'w')
sugarGroupPath = '/{0}/{1}'.format(coverageName,parameterName)
sugarGroup = sugarFile.create_group(sugarGroupPath)
# Create the HDf5 dataset that represents one brick
sugarCubes = sugarGroup.create_dataset('Brick{0}'.format(sugarBrick), sugarBrickSize, dtype=parameterData.content.dtype, chunks=sugarCubeSize)
sugarCubes[:] = np.ones(sugarBrickSize)
#sugarCubes[:] = parameterData
log.debug('Size Before Close: {0}'.format(os.path.getsize(sugarFilePath)))
# Add temporal and spatial metadata as attributes
temporalDomainStart = '20120815060030102030' #CCYYMMDDHHMMSSAABBCC
sugarCubes.attrs["temporal_domain_start"] = np.array(['{0}'.format(temporalDomainStart)])
temporalDomainEnd = '20120815063000102030'
sugarCubes.attrs["temporal_domain_end"] = np.array(['{0}'.format(temporalDomainEnd)])
spatialDomainMinX = '-73' # Longitude
spatialDomainMinY = '40' # Latitude
spatialDomainMinZ = '-10000' # Depth below MLLW in meters
sugarCubes.attrs["spatial_domain_min"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
spatialDomainMaxX = '-70' # Longitude
spatialDomainMaxY = '32' # Latitude
spatialDomainMaxZ = '500' # Depth below MLLW in meters
sugarCubes.attrs["spatial_domain_max"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
# Close the HDF5 file that represents one brick
sugarFile.flush()
log.debug('Size After Flush: {0}'.format(os.path.getsize(sugarFilePath)))
sugarFile.close()
log.debug('Size After Close: {0}'.format(os.path.getsize(sugarFilePath)))
sugarMaster['{0}/Brick{1}'.format(sugarGroupPath,sugarBrick)] = h5py.ExternalLink(sugarFileName, '{0}/Brick{1}'.format(sugarGroupPath,sugarBrick))
sugarMasterGroup.attrs["temporal_domain_start"] = np.array(['{0}'.format(temporalDomainStart)])
sugarMasterGroup.attrs["temporal_domain_end"] = np.array(['{0}'.format(temporalDomainEnd)])
sugarMasterGroup.attrs["spatial_domain_min"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
sugarMasterGroup.attrs["spatial_domain_max"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
# Close the master HDF5 file
log.debug('Master File Size Before Close: {0}'.format(os.path.getsize(masterFilePath)))
sugarMaster.close()
log.debug('Master File Size After Close: {0}'.format(os.path.getsize(masterFilePath)))
log.debug('Finished creating many sugar cubes and bricks!')
return True
def DataProductDeltas():
log.debug('Example to create delta datasets based on a master...')
rootPath = 'test_data'
masterFileName = 'master0.hdf5'
deltaFileName = 'delta0.hdf5'
masterFilePath = '{0}/{1}'.format(rootPath,masterFileName)
dpFilePath = '{0}/{1}'.format(rootPath,deltaFileName)
coverageName = 'COV'
# Open HDf5 file for writing out the SimplexCoverage
sugarMaster = HDFLockingFile(masterFilePath, 'w')
sugarMasterGroup = sugarMaster.create_group('/{0}'.format(coverageName))
dpMaster = HDFLockingFile(dpFilePath, 'w')
dpMasterGroup = dpMaster.create_group('/{0}'.format(coverageName))
dpDeltaGroup = dpMaster.create_group('/{0}'.format('DELTAS'))
parameterNames = ['param01']
# Create persistence objects in HDF5
# Loop through each parameter in the coverage
for parameterName in parameterNames:
# TODO: Brick and chunking size is based on the parameter data and is configurable
sugarBrickSize = (1,100,100)
sugarCubeSize = (1,10,10)
# TODO: Calculate the number of bricks required based on coverage extents and target brick size
sugarBricks = np.arange(10)
for sugarBrick in sugarBricks:
sugarFileName = '{0}_{1}_Brick{2}.hdf5'.format(coverageName,parameterName,sugarBrick)
sugarFilePath = '{0}/{1}'.format(rootPath,sugarFileName)
sugarFile = HDFLockingFile(sugarFilePath, 'w')
sugarGroupPath = '/{0}/{1}'.format(coverageName,parameterName)
sugarGroup = sugarFile.create_group(sugarGroupPath)
# Create the HDf5 dataset that represents one brick
sugarCubes = sugarGroup.create_dataset('Brick{0}'.format(sugarBrick), sugarBrickSize, dtype='f', chunks=sugarCubeSize)
sugarCubes[:] = np.ones((1,100,100))
log.debug('Size Before Flush: {0}'.format(os.path.getsize(sugarFilePath)))
# Close the HDF5 file that represents one brick
sugarFile.flush()
log.debug('Size After Flush: {0}'.format(os.path.getsize(sugarFilePath)))
sugarFile.close()
log.debug('Size After Close: {0}'.format(os.path.getsize(sugarFilePath)))
sugarMaster['{0}/Brick{1}'.format(sugarGroupPath,sugarBrick)] = h5py.ExternalLink(sugarFileName, '{0}/Brick{1}'.format(sugarGroupPath,sugarBrick))
dpMaster['{0}/Brick{1}'.format(sugarGroupPath,sugarBrick)] = h5py.ExternalLink(sugarFileName, '{0}/Brick{1}'.format(sugarGroupPath,sugarBrick))
deltaFilePath = '{0}/{1}'.format(rootPath,'delta.hdf5')
deltaFile = HDFLockingFile(deltaFilePath, 'w')
deltaGroupPath = '/{0}/{1}'.format('DELTAS',parameterName)
deltaGroup = deltaFile.create_group(deltaGroupPath)
# Create the HDf5 dataset that represents one brick
sugarGrains = deltaGroup.create_dataset('Brick{0}'.format(sugarBrick), sugarBrickSize, dtype='f', chunks=sugarCubeSize)
sugarGrains[0,0,0] = 2
dpMaster['DELTAS/param01/Brick9'] = h5py.ExternalLink('delta.hdf5', 'DELTAS/param01/Brick9')
log.debug('Size Before Flush: {0}'.format(os.path.getsize(deltaFilePath)))
# Close the HDF5 file that represents one brick
deltaFile.flush()
log.debug('Size After Flush: {0}'.format(os.path.getsize(deltaFilePath)))
deltaFile.close()
log.debug('Size After Close: {0}'.format(os.path.getsize(deltaFilePath)))
m = dpMaster[('/COV/param01/Brick9')]
d = dpMaster[('/DELTAS/param01/Brick9')]
dpMaster.close()
# Close the master HDF5 file
sugarMaster.close()
log.debug('Finished creating many sugar cubes and bricks!')
return (m,d)
def plGenericBricks(pctFill,fillValue):
log.debug('Creating sample HDF5 persistence objects...')
rootPath = 'test_data'
masterFileName = 'sugarMaster.hdf5'
masterFilePath = '{0}/{1}'.format(rootPath,masterFileName)
coverageName = 'COV'
# Open HDf5 file for writing out the SimplexCoverage
sugarMaster = HDFLockingFile(masterFilePath, 'w')
sugarMasterGroup = sugarMaster.create_group('/{0}'.format(coverageName))
# TODO: Create coverage object from netCDF input file
#TODO: Get parameter names, context and data from the coverage
parameterNames = ['param01','param02','param03','param04']
# Create persistence objects in HDF5
# Loop through each parameter in the coverage
for parameterName in parameterNames:
# TODO: Brick and chunking size is based on the parameter data and is configurable
sugarBrickSize = (1,100,100)
sugarCubeSize = (1,10,10)
# TODO: Calculate the number of bricks required based on coverage extents and target brick size
sugarBricks = np.arange(10)
# TODO: Split up all parameter data into brick-sized arrays
for sugarBrick in sugarBricks:
sugarFileName = '{0}_{1}_Brick{2}.hdf5'.format(coverageName,parameterName,sugarBrick)
sugarFilePath = '{0}/{1}'.format(rootPath,sugarFileName)
sugarFile = HDFLockingFile(sugarFilePath, 'w')
sugarGroupPath = '/{0}/{1}'.format(coverageName,parameterName)
sugarGroup = sugarFile.create_group(sugarGroupPath)
#parameterData = np.empty(sugarBrickSize, dtype='f')
#if pctFill > 0:
# fillData = parameterData[:,0:pctFill,0:pctFill]
# fillData[...] = fillValue
# Create the HDf5 dataset that represents one brick
sugarCubes = sugarGroup.create_dataset('Brick{0}'.format(sugarBrick), sugarBrickSize, dtype=parameterData.dtype, chunks=sugarCubeSize)
sugarCubes[:,0:10,0:10] = np.ones((1,10,10))
log.debug('Size Before Flush: {0}'.format(os.path.getsize(sugarFilePath)))
# Add temporal and spatial metadata as attributes
temporalDomainStart = '20120815060030102030' #CCYYMMDDHHMMSSAABBCC
sugarCubes.attrs["temporal_domain_start"] = np.array(['{0}'.format(temporalDomainStart)])
temporalDomainEnd = '20120815063000102030'
sugarCubes.attrs["temporal_domain_end"] = np.array(['{0}'.format(temporalDomainEnd)])
spatialDomainMinX = '-73' # Longitude
spatialDomainMinY = '40' # Latitude
spatialDomainMinZ = '-10000' # Depth below MLLW in meters
sugarCubes.attrs["spatial_domain_min"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
spatialDomainMaxX = '-70' # Longitude
spatialDomainMaxY = '32' # Latitude
spatialDomainMaxZ = '500' # Depth below MLLW in meters
sugarCubes.attrs["spatial_domain_max"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
# Close the HDF5 file that represents one brick
sugarFile.flush()
log.debug('Size After Flush: {0}'.format(os.path.getsize(sugarFilePath)))
sugarFile.close()
log.debug('Size After Close: {0}'.format(os.path.getsize(sugarFilePath)))
sugarMaster['{0}/Brick{1}'.format(sugarGroupPath,sugarBrick)] = h5py.ExternalLink(sugarFileName, '{0}/Brick{1}'.format(sugarGroupPath,sugarBrick))
sugarMasterGroup.attrs["temporal_domain_start"] = np.array(['{0}'.format(temporalDomainStart)])
sugarMasterGroup.attrs["temporal_domain_end"] = np.array(['{0}'.format(temporalDomainEnd)])
sugarMasterGroup.attrs["spatial_domain_min"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
sugarMasterGroup.attrs["spatial_domain_max"] = np.array(['{0},{1},{2}'.format(spatialDomainMinX, spatialDomainMinY, spatialDomainMinZ)])
# Close the master HDF5 file
log.debug('Master File Size Before Close: {0}'.format(os.path.getsize(masterFilePath)))
sugarMaster.close()
log.debug('Master File Size After Close: {0}'.format(os.path.getsize(masterFilePath)))
log.debug('Finished creating many sugar cubes and bricks!')
return sugarCubes