def test_random_extents(self): nblocks = 100 blocksize = 2000 random_file = RandomWalkGenerator( 'noName', nblocks=nblocks, blocksize=blocksize, ) extents = random_file.get_surface_extents() ans = DT.extents(ndims=2, prec=DT.PRECISION, ll_default=(0, 0), ur_default=(1, 1)) assert_array_equal(ans['ll'], extents['ll']) assert_array_equal(ans['ur'], extents['ur']) extents = random_file.get_subsurface_extents() ans = DT.extents(ndims=2, prec=DT.PRECISION, ll_default=(0, 0, 0), ur_default=(1, 1, 1)) assert_array_equal(ans['ll'], extents['ll']) assert_array_equal(ans['ur'], extents['ur'])
def get_extents(self, ndims, spillets=None): if ndims == 2: return DT.extents(ndims=2, prec=DT.PRECISION, ur_default=(1, 1), ll_default=(0, 0)) elif ndims == 3: return DT.extents(ndims=3, prec=DT.PRECISION, ur_default=(1, 1, 1), ll_default=(0, 0, 0)) else: raise PyStochIOError('Invalid number of dimensions selected.')
def add_product(ops_list, reduce_list, cleaner_list, grid_data, product_metadata): product_type = product_metadata[PRODUCT_TYPE] #allocate space for the gridded result and initialize with max int metadata = {'units': 'm', 'long_name': 'maximum on shore oil thickness'} reduce_array = grid_data.allocate(product_name, DT.SPRECISION, metadata=metadata) # Make a temporary array that will be used in the operation map_array = reduce_array.copy() # Calculate cell diagonal: cell_diagonal = grid_data._grid.cell_diagonal coroutine = max_shore_grid_thickness_op(map_array, cell_diagonal) # Append the operation coroutine to the list ops_list.append(coroutine) # use the array allocated in grid data as the result in the reduce method coroutine = max_reduce(reduce_array) # append the reduce coroutine and the argument it will be passed to the reduce list reduce_list.append((coroutine, map_array)) # Now add a reset value for the map array cleaner_list.append((map_array, DT.SPRECISION(0.0)))
def add_product(ops_list, reduce_list, cleaner_list, grid_data, product_metadata): product_type = product_metadata[PRODUCT_TYPE] config = get_config() # This method now uses a quadratic index space cell_depth_range = config[ product_type].products.max_concentration.cell_depth_range #allocate space for the gridded result metadata = {'units': 'g/cm^3', 'long_name': 'Maximum Oil Concentration'} reduce_array = grid_data.allocate(product_name, DT.SPRECISION, metadata=metadata) # Calculate cell area: cell_area = grid_data._grid.cell_area # m**2 # Make a temporary array that will be used in the operation map_array = reduce_array.copy() coroutine = max_concentration_op(map_array, cell_depth_range, cell_area) # Append the operation coroutine to the list ops_list.append(coroutine) coroutine = max_reduce(reduce_array) reduce_list.append((coroutine, map_array)) # Now add a reset value for the map array cleaner_list.append((map_array, DT.SPRECISION(0.0)))
def make_particles(self, iteration=0): particles = numpy.zeros((self.blocksize, ), dtype=DT.IDEAL_PARTICLE) # Grrr - no way to pass the buffer to put random numbers in ? particles['loc'] = util.random_sample(particles['loc'].shape, dtype=DT.PRECISION) particles['prev_loc'] = util.random_sample(particles['loc'].shape, dtype=DT.PRECISION) particles[ 'lifetime'][:] = ( # Cheat and make time a function of block number DT.SPRECISION(iteration)) mass_offset = DT.SPRECISION(0.9) mass_scale = DT.SPRECISION(.1) mass_time_param = DT.SPRECISION(1) / numpy.log10( DT.SPRECISION(iteration + 2)) # 1/log10(i+2) * (rand[0.9 - 1.0) ) ) particles['mass'] = ( mass_time_param * (util.random_sample(*particles['mass'].shape) * mass_scale + mass_offset)) dens_offset = DT.SPRECISION(0.9) dens_scale = DT.SPRECISION(.1) # dens = rand[0.9 - 1.0) particles['density'] = ( numpy.random.rand(*particles['density'].shape) * dens_scale + dens_offset) return particles
def add_product(ops_list, reduce_list, cleaner_list, grid_data, product_metadata): product_type = product_metadata[PRODUCT_TYPE] #allocate space for the gridded result metadata = { 'units': 'm', 'long_name': 'Maximum of spillet thickness', 'fill_value': DT.SPRECISION(0.0) } reduce_array = grid_data.allocate(product_name, DT.SPRECISION, metadata=metadata) # Calculate cell area: cell_area = grid_data._grid.cell_area # Make a temporary array that will be used in the operation map_array = reduce_array.copy() #config = get_config() #advective_dispersion = config[product_type].products.max_of_spillet_thickness.get('advective_dispersion',numpy.nan) #spillet_dispersion = advective_dispersion * 0.2 # m^2/s #coroutine = thickest_spillet_op(map_array, spillet_dispersion) coroutine = thickest_spillet_op(map_array) # Append the operation coroutine to the list ops_list.append(coroutine) coroutine = max_reduce(reduce_array) reduce_list.append((coroutine, map_array)) # Now add a reset value for the map array cleaner_list.append((map_array, DT.SPRECISION(0.0)))
def get_extents(self, ndims, spillets=None): result = DT.extents(ndims=ndims, prec=DT.SPRECISION) pmin = result['ll'][0] pmax = result['ur'][0] pmax[0] = numpy.max(self._record_data["UR Bound"]["Lon"], axis=0) pmax[1] = numpy.max(self._record_data["UR Bound"]["Lat"], axis=0) pmin[0] = numpy.min(self._record_data["LL Bound"]["Lon"], axis=0) pmin[1] = numpy.min(self._record_data["LL Bound"]["Lat"], axis=0) # Add a little margin to the actual value... result['ll'][0] = pmin - 0.001 * abs(pmin) result['ur'][0] = pmax + 0.001 * abs(pmax) return result
def main(): logger.info('Starting main program') # Get system command line arguments options = get_command_line_arguments() if options.cfgpath is not None: config = Config(options.cfgpath) else: for loc in os.curdir, \ os.path.expanduser(os.path.join('~','.pystoch')), \ os.environ.get("PYSTOCH_CONF_DIR",''): cfg_file = os.path.join(loc, 'config.yaml') if os.path.exists(cfg_file): config = Config(cfg_file) break DT(ndims=2, precision=numpy.float32, location_units='LatLon') wf = WorkFlow() # Uses the Config file rather than passing arguments wf.select_grid_method() wf.select_trajectory_files(options.path, options.prefix, None) # Uses the Config file rather than passing arguments wf.setup_products() logger.info('Pystoch setup complete - running now...') wf.run() logger.info('Pystoch run complete - writing output') if mpi_msr: for name, gd in wf._grid_data.iteritems(): logger.info('Writing output for %s products' % name) netcdf_out(options.outfile + '_' + name + '.nc', options.prefix, wf.nsims, name, gd) logger.info('Pystoch run complete')
def grid_function(grid,operators): index_position = numpy.zeros(0,dtype=DT.IVECTOR) # The IJ location of the particle while True: (block, metadata) = (yield) # an array of particle datatype blen = len(block) particle_position = block['loc'] if index_position.shape != particle_position.shape: index_position = numpy.zeros(particle_position.shape,numpy.int32) else: index_position[:]=0 grid.indexof(particle_position, out=index_position) for target in operators: # send: Particles, index_position, weights and metadata for each cell target.send((block,index_position,DT.PRECISION(1.0), metadata))
def add_product(ops_list, reduce_list, cleaner_list, grid_data, product_metadata): product_type = product_metadata[PRODUCT_TYPE] #allocate space for the gridded result metadata = {'units': 'm^3', 'long_name': 'Aggregate Oil Volume'} reduce_array = grid_data.allocate(product_name, DT.SPRECISION, metadata=metadata) # Make a temporary array that will be used in the operation map_array = reduce_array.copy() coroutine = oil_volume_op(map_array) # Append the operation coroutine to the list ops_list.append(coroutine) coroutine = max_reduce(reduce_array) reduce_list.append((coroutine, map_array)) # Now add a reset value for the map array cleaner_list.append((map_array, DT.SPRECISION(0.0)))
import arcinfo except: try: import arceditor except: pass import arcpy import sys, os, struct, numpy import datetime from pystoch.readers.oilmdl_om_reader import OilModelDirectAccessOMReader as OMDAR from pystoch.datatypes import DT DT(ndims=2, precision=numpy.float32, location_units='LatLon') sr = arcpy.SpatialReference( os.path.join( arcpy.GetInstallInfo()["InstallDir"], r"Coordinate Systems/Geographic Coordinate Systems/World/WGS 1984.prj") ) def ReadScenarioFile(sScenarioFile): sGeoLocPath = os.path.split(os.path.split(sScenarioFile)[0])[0] oScenFile = open(sScenarioFile, "r") scenLines = oScenFile.readlines() for i in range(0, len(scenLines)): key = scenLines[i].split("=")[0]
import matplotlib import matplotlib.pyplot as plt from pystoch import util from pystoch.datatypes import DT, Singleton from pystoch.grids import Grid from pystoch.grid_data import GridData from pystoch.coroutines import * from pystoch.readers.oilmdl_reader import OilModelDirectAccessReader import logging logger = logging.getLogger('pystoch.scripts.oilmdl_oil') nSims = 12 fbase = './trajectory_data/oilmap/pystoch_test_case/PYSTOCHTESTCASE_s{0:03}' DT(ndims=2, precision=numpy.float64, location_units='LatLon') extents = numpy.ones(1, DT.EXTENTS) extent_calc = extents_coroutine(extents) # make the data streamer streamer = stream_coroutine(target=extent_calc) for i in xrange(nSims): fname = fbase.format(i + 1) logger.info('Reading from file: "%s"' % fname) file_reader = OilModelDirectAccessReader(fname) streamer.send(file_reader) logger.info('Got Extents: %s' % extents)
def setUp(self): """ Setup test """ DT(ndims=2,precision=numpy.float32,location_units='LatLon')
def grid_function(grid, operators): # Dummy allocations - will be reallocated as needed. index_position = numpy.zeros( 0, dtype=DT.IVECTOR) # The IJ location of the particle prev_index_position = numpy.zeros( 0, dtype=DT.IVECTOR ) # The IJ location of the particle at the previous timestep index_diff = numpy.zeros( 0, dtype=DT.IVECTOR) # The IJ location change between timesteps index_sum = numpy.zeros( 0, dtype=DT.IVECTOR) # The sum of the IJ location change nsamples_per_grid = 3 # Temporary hack: block_number = 0 while True: (block, metadata) = (yield) # an array of particle datatype block_number += 1 blen = len(block) particle_position = block['loc'] prev_particle_position = block['prev_loc'] # the number of dimensions in a position vector ndims = particle_position.shape[1] if index_position.shape != particle_position.shape: index_position = numpy.zeros(particle_position.shape, numpy.int32) else: index_position[:] = 0 if prev_index_position.shape != prev_particle_position.shape: prev_index_position = numpy.zeros(prev_particle_position.shape, numpy.int32) else: prev_index_position[:] = 0 if index_diff.shape != prev_particle_position.shape: index_diff = numpy.zeros(prev_particle_position.shape, numpy.int32) else: index_diff[:] = 0 if index_sum.shape != blen: index_sum = numpy.zeros(blen, numpy.int32) else: index_sum[:] = 0 # Calculate the IJ index of each particle now and previously grid.indexof(particle_position, out=index_position) grid.indexof(prev_particle_position, out=prev_index_position) #evaluate the absolute value of the index space difference index_diff[...] = numpy.abs(index_position - prev_index_position) index_sum[...] = numpy.sum(index_diff, axis=1) for i in xrange(blen): if index_sum[i] > 0: delta_pos = particle_position[i, :] - prev_particle_position[ i, :] samples = index_sum[i] * nsamples_per_grid interpolated_positions = delta_pos.reshape( 1, ndims) * numpy.arange(samples).reshape( samples, 1) / DT.PRECISION(samples) interpolated_positions += prev_particle_position[i, :].reshape( 1, ndims) interpolated_index_positions = grid.indexof( interpolated_positions) # Make the weight array the correct size weight = DT.PRECISION(1.0 / samples) * numpy.ones((samples, )) for target in operators: target.send((block[i], interpolated_index_positions, weight, metadata)) else: # Make the weight array the correct size weight = numpy.ones((1, )) for target in operators: target.send( (block[i], prev_index_position[i, :].reshape(1, ndims), weight, metadata))
def _load_meta(self): record_data = None header_data = None version_number = None with open('.'.join([self.fname, self.RECORD_FILE_SUFFIX]), 'rb') as pcl_file: # Read the header record header_data = numpy.fromfile( pcl_file, dtype=self.SPLMDL_PCL_HEADER_RECORD_TYPE, count=1) # Read the rest of the entries record_data = numpy.fromfile( pcl_file, dtype=self.SPLMDL_PCL_LOOKUP_RECORD_TYPE, count=-1) if header_data is None: raise PyStocheIOError( "Could not read the header from the file: '%s'" % '.'.join([self.fname, self.RECORD_FILE_SUFFIX])) version_number = header_data[0]['pcd_version'] if not version_number in self.supported_versions: raise NotImplementedError('Can not read pcd file version "%d"' % version_number) self._version_number = version_number if record_data is None: raise PyStochIOError( "Could not read data from PCL file:'%s'" % '.'.join([self.fname, self.RECORD_FILE_SUFFIX])) if len(record_data) == 0: raise PyStochIOError( "PCL file '%s' is empty - no records!" % '.'.join([self.fname, self.RECORD_FILE_SUFFIX])) ### CONVERT FROM FORTRAN TO C INDEXING! ### record_data['record_start'] -= 1 self._record_data = record_data npseudo = header_data[0]['npseudo'] self.set_record_size(version_number, npseudo) #logger.info('record_header: \n%s' % header_data) #logger.info('record_lookup: \n%s' % record_data) if version_number == -11: lookup_size = 19 * 4 self._SPLMDL_PCD_LOOKUP_RECORD_TYPE = numpy.dtype([ ('surface_records', numpy.int32, 1), ('subsurface_dissolved_records', numpy.int32, 1), ('subsurface_residual_records', numpy.int32, 1), ('tarball_records', numpy.int32, 1), ('sediment_records', numpy.int32, 1), ('shoreline_records', numpy.int32, 1), ('wave_height', DT.SPRECISION, 1), ('wind_average', DT.SPRECISION, 1), ('velocity_max', DT.SPRECISION, 1), ('surface_extent', DT.extent_type(ndims=2, prec=DT.SPRECISION), 1), #Must reorder from: #XMIN, XMAX, YMIN, YMAX ('subsurface_extent', DT.extent_type(ndims=3, prec=DT.SPRECISION), 1), #Must reorder from: #XMIN, XMAX, YMIN, YMAX, ZMIN, ZMAX ### Unknown number of psuedo components - must be defined when the header is read! ('junk', numpy.character, self.PCD_RECORD_SIZE - lookup_size), ]) elif version_number == -10: lookup_size = 18 * 4 self._SPLMDL_PCD_LOOKUP_RECORD_TYPE = numpy.dtype([ ('surface_records', numpy.int32, 1), ('subsurface_dissolved_records', numpy.int32, 1), ('subsurface_residual_records', numpy.int32, 1), ('sediment_records', numpy.int32, 1), ('shoreline_records', numpy.int32, 1), ('wave_height', DT.SPRECISION, 1), ('wind_average', DT.SPRECISION, 1), ('velocity_max', DT.SPRECISION, 1), ('surface_extent', DT.extent_type(ndims=2, prec=DT.SPRECISION), 1), #Must reorder from: #XMIN, XMAX, YMIN, YMAX ('subsurface_extent', DT.extent_type(ndims=3, prec=DT.SPRECISION), 1), #Must reorder from: #XMIN, XMAX, YMIN, YMAX, ZMIN, ZMAX ### Unknown number of psuedo components - must be defined when the header is read! ('junk', numpy.character, self.PCD_RECORD_SIZE - lookup_size), ]) else: # -8 or -9 lookup_size = 8 * 4 self._SPLMDL_PCD_LOOKUP_RECORD_TYPE = numpy.dtype([ ('surface_records', numpy.int32, 1), ('subsurface_dissolved_records', numpy.int32, 1), ('subsurface_residual_records', numpy.int32, 1), ('sediment_records', numpy.int32, 1), ('shoreline_records', numpy.int32, 1), ('wave_height', DT.SPRECISION, 1), ('wind_average', DT.SPRECISION, 1), ('velocity_max', DT.SPRECISION, 1), ### Unknown number of psuedo components - must be defined when the header is read! ('junk', numpy.character, self.PCD_RECORD_SIZE - lookup_size), ]) # particle_type is 0 SPLMDL_SURFACE_PARTICLE = self.create_surface_spillet_type( version_number, npseudo) # particle_type is -1 SPLMDL_SUBSURFACE_DISSOLVED_PARTICLE = self.create_subsurface_dissolved_spillet_type( version_number, npseudo) # particle_type is -2 SPLMDL_SUBSURFACE_RESIDUAL_PARTICLE = self.create_subsurface_residual_spillet_type( version_number, npseudo) # particle_type is -3 SPLMDL_TARBALL_PARTICLE = self.create_tarball_spillet_type( version_number, npseudo) # particle_type is -12 SPLMDL_SEDIMENT_PARTICLE = self.create_sediment_spillet_type( version_number, npseudo) SPLMDL_SHORELINE_PARTICLE = self.create_shoreline_spillet_type( version_number, npseudo) self.spillet_types = { SplModelDirectAccessReader.SURFACE_SPILLETS: { 'dtype': SPLMDL_SURFACE_PARTICLE, 'offset': None, 'count': None }, SplModelDirectAccessReader.SHORELINE_SPILLETS: { 'dtype': SPLMDL_SHORELINE_PARTICLE, 'offset': None, 'count': None }, SplModelDirectAccessReader.SUBSURFACE_DISSOLVED_SPILLETS: { 'dtype': SPLMDL_SUBSURFACE_DISSOLVED_PARTICLE, 'offset': None, 'count': None }, SplModelDirectAccessReader.SUBSURFACE_RESIDUAL_SPILLETS: { 'dtype': SPLMDL_SUBSURFACE_RESIDUAL_PARTICLE, 'offset': None, 'count': None }, SplModelDirectAccessReader.TARBALL_SPILLETS: { 'dtype': SPLMDL_TARBALL_PARTICLE, 'offset': None, 'count': None }, SplModelDirectAccessReader.SEDIMENT_SPILLETS: { 'dtype': SPLMDL_SEDIMENT_PARTICLE, 'offset': None, 'count': None }, } if version_number <= -11 and SplModelDirectAccessReader.TARBALL_SPILLETS not in self.product_spillet_map[ SURFACE]: # Modify the class variable based on whether this is version 11 or not... # Not a great way to deal with this. Better ideas? self.product_spillet_map[SURFACE].append( SplModelDirectAccessReader.TARBALL_SPILLETS) self.product_spillet_map[SUBSURFACE].append( SplModelDirectAccessReader.TARBALL_SPILLETS)
def setUp(self): """ Setup test """ Singleton._instances.clear() DT(ndims=2, precision=numpy.float32, location_units='LatLon')